http body handling: use streaming buffer API

Convert HTTP body handling to use the Streaming Buffer API. This means
the HtpBodyChunks no longer maintain their own data segments, but
instead add their data to the StreamingBuffer instance in the HtpBody
structure.

In case the HtpBodyChunk needs to access it's data it can do so still
through the Streaming Buffer API.

Updates & simplifies the various users of the reassembled bodies:
multipart parsing and the detection engine.

diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c
index d3720f8d39bb87f190df9a9cf908b11fac960085..53194307b4b857018e48b7619935cefdd5bb539b 100644 (file)
--- a/src/app-layer-htp-body.c
+++ b/src/app-layer-htp-body.c
@@ -62,6 +62,9 @@
 
 #include "util-memcmp.h"
 
+static StreamingBufferConfig default_cfg = {
+    0, 0, 3072, HTPMalloc, HTPCalloc, HTPRealloc, HTPFree };
+
 /**
  * \brief Append a chunk of body to the HtpBody struct
  *
@@ -82,20 +85,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         SCReturnInt(0);
     }
 
+    if (body->sb == NULL) {
+        body->sb = StreamingBufferInit(&default_cfg);
+        if (body->sb == NULL)
+            SCReturnInt(-1);
+    }
+
     if (body->first == NULL) {
         /* New chunk */
         bd = (HtpBodyChunk *)HTPCalloc(1, sizeof(HtpBodyChunk));
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = 0;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->first = body->last = bd;
 
@@ -105,29 +107,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = body->content_len_so_far;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->last->next = bd;
         body->last = bd;
 
         body->content_len_so_far += len;
     }
-    SCLogDebug("Body %p; data %p, len %"PRIu32, body, bd->data, (uint32_t)bd->len);
+    SCLogDebug("body %p", body);
 
     SCReturnInt(0);
 
 error:
     if (bd != NULL) {
-        if (bd->data != NULL) {
-            HTPFree(bd->data, bd->len);
-        }
         HTPFree(bd, sizeof(HtpBodyChunk));
     }
     SCReturnInt(-1);
@@ -150,9 +142,12 @@ void HtpBodyPrint(HtpBody *body)
         SCLogDebug("--- Start body chunks at %p ---", body);
         printf("--- Start body chunks at %p ---\n", body);
         for (cur = body->first; cur != NULL; cur = cur->next) {
-            SCLogDebug("Body %p; data %p, len %"PRIu32, body, cur->data, (uint32_t)cur->len);
-            printf("Body %p; data %p, len %"PRIu32"\n", body, cur->data, (uint32_t)cur->len);
-            PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len);
+            const uint8_t *data = NULL;
+            uint32_t data_len = 0;
+            StreamingBufferSegmentGetData(body->sb, &cur->sbseg, &data, &data_len);
+            SCLogDebug("Body %p; data %p, len %"PRIu32, body, data, data_len);
+            printf("Body %p; data %p, len %"PRIu32"\n", body, data, data_len);
+            PrintRawDataFp(stdout, data, data_len);
         }
         SCLogDebug("--- End body chunks at %p ---", body);
     }
@@ -170,8 +165,7 @@ void HtpBodyFree(HtpBody *body)
     if (body->first == NULL)
         return;
 
-    SCLogDebug("Removing chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    SCLogDebug("removing chunks of body %p", body);
 
     HtpBodyChunk *cur = NULL;
     HtpBodyChunk *prev = NULL;
@@ -179,12 +173,12 @@ void HtpBodyFree(HtpBody *body)
     prev = body->first;
     while (prev != NULL) {
         cur = prev->next;
-        if (prev->data != NULL)
-            HTPFree(prev->data, prev->len);
         HTPFree(prev, sizeof(HtpBodyChunk));
         prev = cur;
     }
     body->first = body->last = NULL;
+
+    StreamingBufferFree(body->sb);
 }
 
 /**
@@ -230,24 +224,26 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
         SCReturn;
     }
 
-    SCLogDebug("Pruning chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    uint64_t left_edge = body->body_inspected;
+    if (left_edge <= min_size || left_edge <= window)
+        left_edge = 0;
+    if (left_edge)
+        left_edge -= window;
+
+    if (left_edge) {
+        SCLogDebug("sliding body to offset %"PRIu64, left_edge);
+        StreamingBufferSlideToOffset(body->sb, left_edge);
+    }
+
+    SCLogDebug("pruning chunks of body %p", body);
 
     HtpBodyChunk *cur = body->first;
     while (cur != NULL) {
         HtpBodyChunk *next = cur->next;
+        SCLogDebug("cur %p", cur);
 
-        SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", "
-                "body->body_parsed %"PRIu64, cur->stream_offset, cur->len,
-                cur->stream_offset + cur->len, body->body_parsed);
-
-        uint64_t left_edge = body->body_inspected;
-        if (left_edge <= min_size || left_edge <= window)
-            left_edge = 0;
-        if (left_edge)
-            left_edge -= window;
-
-        if (cur->stream_offset + cur->len > left_edge) {
+        if (!StreamingBufferSegmentIsBeforeWindow(body->sb, &cur->sbseg)) {
+            SCLogDebug("not removed");
             break;
         }
 
@@ -256,12 +252,10 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
             body->last = next;
         }
 
-        if (cur->data != NULL) {
-            HTPFree(cur->data, cur->len);
-        }
         HTPFree(cur, sizeof(HtpBodyChunk));
 
         cur = next;
+        SCLogDebug("removed");
     }
 
     SCReturn;

diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 9e68f097085cbf360611af248e76a6229f9be4a2..33b747f13e95b7feca22632d4c1eb4dbf0dde759 100644 (file)
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -629,8 +629,11 @@ static int HTPFileParserTest03(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
+    if (http_state->files_ts->head->chunks_head == NULL ||
+        http_state->files_ts->head->chunks_head->len != 11)
+    {
+        if (http_state->files_ts->head->chunks_head != NULL)
+            printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
         goto end;
     }
 

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 45c14d7d81fee1ecdc7300fd79595cfecc2608ad..f8069c757b7b5b77fe7f9a0ff165c45294889261 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1170,60 +1170,11 @@ static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
  *  \param chunks_buffer_len pointer to pass back the buffer length to the caller
  */
 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
-        uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
+        const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
 {
-    uint8_t *buf = NULL;
-    uint8_t *pbuf = NULL;
-    uint32_t buf_len = 0;
-    HtpBodyChunk *cur = htud->request_body.first;
-
-    for ( ; cur != NULL; cur = cur->next) {
-        SCLogDebug("chunk %p", cur);
-
-        /* skip body chunks entirely before what we parsed already */
-        if ((uint64_t )cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
-            SCLogDebug("skipping chunk");
-            continue;
-        }
-
-        SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64,
-            cur->stream_offset, cur->len, htud->request_body.body_parsed);
-
-        if (cur->stream_offset < htud->request_body.body_parsed &&
-                cur->stream_offset + cur->len >= htud->request_body.body_parsed) {
-            SCLogDebug("use part");
-
-            uint32_t toff = htud->request_body.body_parsed - cur->stream_offset;
-            uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed;
-            uint8_t *pbuf = NULL;
-
-            buf_len += tlen;
-            if ((pbuf = HTPRealloc(buf, buf_len - tlen, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - tlen);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - tlen, cur->data + toff, tlen);
-
-        } else {
-            SCLogDebug("use entire chunk");
-
-            buf_len += cur->len;
-            if ((pbuf = HTPRealloc(buf, buf_len - cur->len, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - cur->len);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - cur->len, cur->data, cur->len);
-        }
-    }
-
-    *chunks_buffer = buf;
-    *chunks_buffer_len = buf_len;
+    StreamingBufferGetDataAtOffset(htud->request_body.sb,
+            chunks_buffer, chunks_buffer_len,
+            htud->request_body.body_parsed);
 }
 
 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
@@ -1249,8 +1200,8 @@ static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
     memcpy(boundary + 2, htud->boundary, htud->boundary_len);
 }
 
-int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
-                                  void *tx, uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
+int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
+        const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
 {
     int result = 0;
     uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
@@ -1288,7 +1239,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
         if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
             SCLogDebug("reached the end of the file");
 
-            uint8_t *filedata = chunks_buffer;
+            const uint8_t *filedata = chunks_buffer;
             uint32_t filedata_len = 0;
             uint8_t flags = 0;
 
@@ -1328,7 +1279,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
             SCLogDebug("not yet at the end of the file");
 
             if (chunks_buffer_len > expected_boundary_end_len) {
-                uint8_t *filedata = chunks_buffer;
+                const uint8_t *filedata = chunks_buffer;
                 uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
 #ifdef PRINT
                 printf("FILEDATA (part) START: \n");
@@ -1816,7 +1767,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         HtpBodyAppendChunk(&tx_ud->request_body, d->data, len);
 
-        uint8_t *chunks_buffer = NULL;
+        const uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
 
         if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
@@ -1837,9 +1788,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
             HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
 
-            if (chunks_buffer != NULL) {
-                HTPFree(chunks_buffer, chunks_buffer_len);
-            }
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
             HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
@@ -2189,6 +2137,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
 #endif
     cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE;
 
+    cfg_prec->sbcfg.flags = 0;
+    cfg_prec->sbcfg.buf_size = cfg_prec->request_inspect_window;
+    cfg_prec->sbcfg.buf_slide = 0;
+
     htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
@@ -5666,7 +5618,7 @@ static int HTPBodyReassemblyTest01(void)
     r = HtpBodyAppendChunk(&htud.request_body, chunk2, sizeof(chunk2)-1);
     BUG_ON(r != 0);
 
-    uint8_t *chunks_buffer = NULL;
+    const uint8_t *chunks_buffer = NULL;
     uint32_t chunks_buffer_len = 0;
 
     HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index d7aab939937207e6a2c08457eee868ea58b90441..3c7ae66cbf17aa8880bbb6ab11e42ab2e62fddd5 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -37,6 +37,7 @@
 #include "util-file.h"
 #include "app-layer-htp-mem.h"
 #include "detect-engine-state.h"
+#include "util-streaming-buffer.h"
 
 #include <htp/htp.h>
 
@@ -158,14 +159,14 @@ typedef struct HTPCfgRec_ {
     int                 randomize;
     int                 randomize_range;
     int                 http_body_inline;
+
+    StreamingBufferConfig sbcfg;
 } HTPCfgRec;
 
 /** Struct used to hold chunks of a body on a request */
 struct HtpBodyChunk_ {
-    uint8_t *data;              /**< Pointer to the data of the chunk */
     struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */
-    uint64_t stream_offset;
-    uint32_t len;               /**< Length of the chunk */
+    StreamingBufferSegment sbseg;
     int logged;
 } __attribute__((__packed__));
 typedef struct HtpBodyChunk_ HtpBodyChunk;
@@ -175,6 +176,8 @@ typedef struct HtpBody_ {
     HtpBodyChunk *first; /**< Pointer to the first chunk */
     HtpBodyChunk *last;  /**< Pointer to the last chunk */
 
+    StreamingBuffer *sb;
+
     /* Holds the length of the htp request body seen so far */
     uint64_t content_len_so_far;
     /* parser tracker */

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 5fc48c5fc0578108f5f3ed9f0dafac8c5558f74b..1b4b1534a3d1bbba7fcf11ecee1d9db220cc61a1 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -97,7 +97,7 @@ static inline int HCBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
 
 /**
  */
-static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -106,7 +106,7 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -170,49 +170,12 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         goto end;
     }
 
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->request_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->request_body.body_inspected) {
-                if ((htud->request_body.body_inspected - cur->stream_offset) > htp_state->cfg->request_inspect_min_size) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hcbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hcbd[index].buffer == NULL || (det_ctx->hcbd[index].buffer_len + cur->len) > det_ctx->hcbd[index].buffer_size) {
-            void *ptmp;
-            det_ctx->hcbd[index].buffer_size += cur->len * 2;
-
-            if ((ptmp = SCRealloc(det_ctx->hcbd[index].buffer, det_ctx->hcbd[index].buffer_size)) == NULL) {
-                SCFree(det_ctx->hcbd[index].buffer);
-                det_ctx->hcbd[index].buffer = NULL;
-                det_ctx->hcbd[index].buffer_size = 0;
-                det_ctx->hcbd[index].buffer_len = 0;
-                goto end;
-            }
-            det_ctx->hcbd[index].buffer = ptmp;
-        }
-        memcpy(det_ctx->hcbd[index].buffer + det_ctx->hcbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hcbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
+    StreamingBufferGetData(htud->request_body.sb,
+            &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len,
+            &det_ctx->hcbd[index].offset);
 
     /* update inspected tracker */
-    htud->request_body.body_inspected = htud->request_body.last->stream_offset + htud->request_body.last->len;
+    htud->request_body.body_inspected = htud->request_body.last->sbseg.stream_offset + htud->request_body.last->sbseg.segment_len;
 
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
@@ -258,7 +221,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     uint32_t cnt = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -267,7 +230,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpClientBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpClientBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -282,7 +245,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -296,7 +259,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL);

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 48648e3f4e3b91daaf0b6bb791e549be76937dd5..3918c6e094c3f26646d9af74087dad79b202f06f 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -96,134 +96,7 @@ static inline int HSBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
     return 0;
 }
 
-static void HSBDGetBufferForTXInIDSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->response_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->response_body.body_inspected) {
-                if ((htud->response_body.body_inspected - cur->stream_offset) > htp_state->cfg->response_inspect_window) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hsbd[index].buffer == NULL || (det_ctx->hsbd[index].buffer_len + cur->len) > det_ctx->hsbd[index].buffer_size) {
-            void *ptmp;
-            uint32_t newsize = det_ctx->hsbd[index].buffer_size + (cur->len * 2);
-
-            if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, newsize)) == NULL) {
-                HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-                det_ctx->hsbd[index].buffer = NULL;
-                det_ctx->hsbd[index].buffer_size = 0;
-                det_ctx->hsbd[index].buffer_len = 0;
-                return;
-            }
-            det_ctx->hsbd[index].buffer = ptmp;
-            det_ctx->hsbd[index].buffer_size = newsize;
-        }
-        memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hsbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker */
-    htud->response_body.body_inspected = htud->response_body.last->stream_offset + htud->response_body.last->len;
-}
-
-#define MAX_WINDOW 10*1024*1024
-static void HSBDGetBufferForTXInIPSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    uint32_t window_size = 0;
-
-    /* how much from before body_inspected will we consider? */
-    uint32_t cfg_win =
-        htud->response_body.body_inspected >= htp_state->cfg->response_inspect_min_size ?
-            htp_state->cfg->response_inspect_window :
-            htp_state->cfg->response_inspect_min_size;
-
-    /* but less if we don't have that much before body_inspected */
-    if ((htud->response_body.body_inspected - htud->response_body.first->stream_offset) < cfg_win) {
-        cfg_win = htud->response_body.body_inspected - htud->response_body.first->stream_offset;
-    }
-    window_size = (htud->response_body.content_len_so_far - htud->response_body.body_inspected) + cfg_win;
-    if (window_size > MAX_WINDOW) {
-        SCLogDebug("weird: body size is %uk", window_size/1024);
-        window_size = MAX_WINDOW;
-    }
-
-    if (det_ctx->hsbd[index].buffer == NULL || window_size > det_ctx->hsbd[index].buffer_size) {
-        void *ptmp;
-
-        if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, window_size)) == NULL) {
-            HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-            det_ctx->hsbd[index].buffer = NULL;
-            det_ctx->hsbd[index].buffer_size = 0;
-            det_ctx->hsbd[index].buffer_len = 0;
-            return;
-        }
-        det_ctx->hsbd[index].buffer = ptmp;
-        det_ctx->hsbd[index].buffer_size = window_size;
-    }
-
-    uint32_t left_edge = htud->response_body.body_inspected - cfg_win;
-
-    int first = 1;
-    while (cur != NULL) {
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* entirely before our window */
-        if ((cur->stream_offset + cur->len) <= left_edge) {
-            cur = cur->next;
-            continue;
-        } else {
-            uint32_t offset = 0;
-            if (cur->stream_offset < left_edge && (cur->stream_offset + cur->len) > left_edge) {
-                offset = left_edge - cur->stream_offset;
-                BUG_ON(offset > cur->len);
-            }
-
-            /* unusual: if window isn't big enough, we just give up */
-            if (det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size) {
-                htud->response_body.body_inspected = cur->stream_offset;
-                SCReturn;
-            }
-
-            BUG_ON(det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size);
-
-            memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data + offset, cur->len - offset);
-            det_ctx->hsbd[index].buffer_len += (cur->len - offset);
-            det_ctx->hsbd[index].offset -= offset;
-        }
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker to point before the current window */
-    htud->response_body.body_inspected = htud->response_body.content_len_so_far;
-}
-
-static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -232,7 +105,7 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -303,14 +176,25 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                        "entire body.");
             goto end;
         }
-        HSBDGetBufferForTXInIDSMode(det_ctx, htp_state, cur, htud, index);
+    }
+
+    StreamingBufferGetData(htud->response_body.sb,
+            &det_ctx->hsbd[index].buffer, &det_ctx->hsbd[index].buffer_len,
+            &det_ctx->hsbd[index].offset);
+
+    /* update inspected tracker */
+    if (!htp_state->cfg->http_body_inline) {
+        htud->response_body.body_inspected = htud->response_body.last->sbseg.stream_offset +
+                                             htud->response_body.last->sbseg.segment_len;
     } else {
-        HSBDGetBufferForTXInIPSMode(det_ctx, htp_state, cur, htud, index);
+        htud->response_body.body_inspected = htud->response_body.content_len_so_far;
     }
+    SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
 
     buffer = det_ctx->hsbd[index].buffer;
     *buffer_len = det_ctx->hsbd[index].buffer_len;
     *stream_start_offset = det_ctx->hsbd[index].offset;
+
  end:
     return buffer;
 }
@@ -361,7 +245,7 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpServerBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpServerBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -377,7 +261,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -391,7 +275,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL);

diff --git a/src/detect-engine.c b/src/detect-engine.c
index 34f15652023ba7a3cf44f67537597d2ebf0d1a2b..f591056848fa91540d92877eaa265568669287fb 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -1687,22 +1687,12 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     /* HSBD */
     if (det_ctx->hsbd != NULL) {
         SCLogDebug("det_ctx hsbd %u", det_ctx->hsbd_buffers_size);
-        for (i = 0; i < det_ctx->hsbd_buffers_size; i++) {
-            if (det_ctx->hsbd[i].buffer != NULL) {
-                HTPFree(det_ctx->hsbd[i].buffer, det_ctx->hsbd[i].buffer_size);
-            }
-        }
         SCFree(det_ctx->hsbd);
     }
 
     /* HSCB */
     if (det_ctx->hcbd != NULL) {
         SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_size);
-        for (i = 0; i < det_ctx->hcbd_buffers_size; i++) {
-            if (det_ctx->hcbd[i].buffer != NULL)
-                SCFree(det_ctx->hcbd[i].buffer);
-            SCLogDebug("det_ctx->hcbd[i].buffer_size %u", det_ctx->hcbd[i].buffer_size);
-        }
         SCFree(det_ctx->hcbd);
     }
 

diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index 729629a098bafacc4ce49b957d85bf639a90b477..2813d4ca995d0d6393961a3ea93e0bb68f6b241e 100644 (file)
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -1641,7 +1641,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body one!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -1654,7 +1656,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body two!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }

diff --git a/src/detect-pcre.c b/src/detect-pcre.c
index b4303744f4a7f327f7a1af669f356555effc812c..72692ee0734ae0ce9288e70637066dad4b6f8ae6 100644 (file)
--- a/src/detect-pcre.c
+++ b/src/detect-pcre.c
@@ -3116,7 +3116,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3129,7 +3129,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3358,7 +3358,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3371,7 +3371,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }

diff --git a/src/detect.h b/src/detect.h
index 90c6502984259bdcce27ce8d481c95b06d9e0bb9..8a57b08161657019c81f0bb8ba35684aa817b936 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -697,7 +697,7 @@ enum {
 };
 
 typedef struct HttpReassembledBody_ {
-    uint8_t *buffer;
+    const uint8_t *buffer;
     uint32_t buffer_size;   /**< size of the buffer itself */
     uint32_t buffer_len;    /**< data len in the buffer */
     uint64_t offset;        /**< data offset */

diff --git a/src/output-streaming.c b/src/output-streaming.c
index 4ae7c58c2e5e18e5957294226cb50dad0cd8ce83..234fbee70ebd968b5d64ba9f95ee28568a637738 100644 (file)
--- a/src/output-streaming.c
+++ b/src/output-streaming.c
@@ -205,7 +205,7 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                         }
 
                         uint8_t flags = iflags | OUTPUT_STREAMING_FLAG_TRANSACTION;
-                        if (chunk->stream_offset == 0)
+                        if (chunk->sbseg.stream_offset == 0)
                             flags |= OUTPUT_STREAMING_FLAG_OPEN;
                         /* if we need to close and we're at the last segment in the list
                          * we add the 'close' flag so the logger can close up. */
@@ -213,9 +213,13 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                             flags |= OUTPUT_STREAMING_FLAG_CLOSE;
                         }
 
+                        const uint8_t *data = NULL;
+                        uint32_t data_len = 0;
+                        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+
                         // invoke Streamer
-                        Streamer(cbdata, f, chunk->data, (uint32_t)chunk->len, tx_id, flags);
-                        //PrintRawDataFp(stdout, chunk->data, chunk->len);
+                        Streamer(cbdata, f, data, data_len, tx_id, flags);
+                        //PrintRawDataFp(stdout, data, data_len);
                         chunk->logged = 1;
                         tx_logged = 1;
                     }

diff --git a/src/util-lua-http.c b/src/util-lua-http.c
index 3d97b0f640acd365c0289a590e2c5b42fb27752c..1207f7f9ef3ede8946e07e6f14210456789a914f 100644 (file)
--- a/src/util-lua-http.c
+++ b/src/util-lua-http.c
@@ -285,7 +285,12 @@ static int HttpGetBody(lua_State *luastate, int dir)
     lua_newtable(luastate);
     while (chunk != NULL) {
         lua_pushinteger(luastate, index);
-        LuaPushStringBuffer(luastate, chunk->data, chunk->len);
+
+        const uint8_t *data = NULL;
+        uint32_t data_len = 0;
+        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+        LuaPushStringBuffer(luastate, data, data_len);
+
         lua_settable(luastate, -3);
 
         chunk = chunk->next;
@@ -293,8 +298,8 @@ static int HttpGetBody(lua_State *luastate, int dir)
     }
 
     if (body->first && body->last) {
-        lua_pushinteger(luastate, body->first->stream_offset);
-        lua_pushinteger(luastate, body->last->stream_offset + body->last->len);
+        lua_pushinteger(luastate, body->first->sbseg.stream_offset);
+        lua_pushinteger(luastate, body->last->sbseg.stream_offset + body->last->sbseg.segment_len);
         return 3;
     } else {
         return 1;