From: Victor Julien <victor@inliniac.net>
Date: Fri, 17 Feb 2017 16:59:43 +0000 (+0100)
Subject: detect / stream: new 'raw' stream inspection
X-Git-Tag: suricata-4.0.0-beta1~171
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=971ab18b95fad189f863ef7f201c458e43540139;p=thirdparty%2Fsuricata.git

detect / stream: new 'raw' stream inspection

Remove the 'StreamMsg' approach from the engine. In this approach the
stream engine would create a list of chunks for inspection by the
detection engine. There were several issues:

1. the messages had a fixed size, so blocks of data bigger than ~4k
   would be cut into multiple messages

2. it lead to lots of data copying and unnecessary memory use

3. the StreamMsgs used a central pool

The Stream engine switched over to the streaming buffer API, which
means that the reassembled data is always available. This made the
StreamMsg approach even clunkier.

The new approach exposes the streaming buffer data to the detection
engine. It has to pay attention to an important issue though: packet
loss. The data may have gaps. The streaming buffer API tracks the
blocks of continuous data.

To access the data for inspection a callback approach is used. The
'StreamReassembleRaw' function is called with a callback and data.
This way it runs the MPM and individual rule inspection code. At
the end of each detection run the stream engine is notified that it
can move forward it's 'progress'.
---

diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c
index 338d6e8149..db7b911088 100644
--- a/src/app-layer-detect-proto.c
+++ b/src/app-layer-detect-proto.c
@@ -3679,110 +3679,6 @@ static int AppLayerProtoDetectTest19(void)
     return result;
 }
 
-/** \test test if the engine detect the proto and match with it
- *        and also against a content option */
-static int AppLayerProtoDetectTest20(void)
-{
-    int result = 0;
-    Flow *f = NULL;
-    uint8_t http_buf1[] = "POST /one HTTP/1.0\r\n"
-        "User-Agent: Mozilla/1.0\r\n"
-        "Cookie: hellocatch\r\n\r\n";
-    uint32_t http_buf1_len = sizeof(http_buf1) - 1;
-    TcpSession ssn;
-    Packet *p = NULL;
-    Signature *s = NULL;
-    ThreadVars tv;
-    DetectEngineThreadCtx *det_ctx = NULL;
-    DetectEngineCtx *de_ctx = NULL;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&tv, 0, sizeof(ThreadVars));
-    memset(&ssn, 0, sizeof(TcpSession));
-
-    p = UTHBuildPacket(http_buf1, http_buf1_len, IPPROTO_TCP);
-
-    f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
-    if (f == NULL)
-        goto end;
-    f->protoctx = &ssn;
-    p->flow = f;
-    p->flowflags |= FLOW_PKT_TOSERVER;
-    p->flowflags |= FLOW_PKT_ESTABLISHED;
-    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
-    f->alproto = ALPROTO_HTTP;
-    f->proto = IPPROTO_TCP;
-    p->flags |= PKT_STREAM_ADD;
-    p->flags |= PKT_STREAM_EOF;
-
-    de_ctx = DetectEngineCtxInit();
-    if (de_ctx == NULL) {
-        goto end;
-    }
-
-    StreamTcpInitConfig(TRUE);
-
-    StreamMsg *stream_msg = StreamMsgGetFromPool();
-    if (stream_msg == NULL) {
-        printf("no stream_msg: ");
-        goto end;
-    }
-
-    memcpy(stream_msg->data, http_buf1, http_buf1_len);
-    stream_msg->data_len = http_buf1_len;
-
-    ssn.toserver_smsg_head = stream_msg;
-    ssn.toserver_smsg_tail = stream_msg;
-
-    de_ctx->flags |= DE_QUIET;
-
-    s = de_ctx->sig_list = SigInit(de_ctx, "alert http any any -> any any "
-                                   "(msg:\"Test content option\"; "
-                                   "content:\"one\"; sid:1;)");
-    if (s == NULL) {
-        goto end;
-    }
-
-    SigGroupBuild(de_ctx);
-    DetectEngineThreadCtxInit(&tv, (void *)de_ctx, (void *)&det_ctx);
-
-    FLOWLOCK_WRLOCK(f);
-    int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
-                                STREAM_TOSERVER, http_buf1, http_buf1_len);
-    if (r != 0) {
-        printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(f);
-
-    /* do detect */
-    SigMatchSignatures(&tv, de_ctx, det_ctx, p);
-
-    if (!PacketAlertCheck(p, 1)) {
-        printf("sig 1 didn't alert, but it should: ");
-        goto end;
-    }
-
-    result = 1;
-
- end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    if (det_ctx != NULL)
-        DetectEngineThreadCtxDeinit(&tv, det_ctx);
-    if (de_ctx != NULL)
-        SigGroupCleanup(de_ctx);
-    if (de_ctx != NULL)
-        DetectEngineCtxFree(de_ctx);
-
-    StreamTcpFreeConfig(TRUE);
-    UTHFreePackets(&p, 1);
-    UTHFreeFlow(f);
-    return result;
-}
-
-
 void AppLayerProtoDetectUnittestsRegister(void)
 {
     SCEnter();
@@ -3806,7 +3702,6 @@ void AppLayerProtoDetectUnittestsRegister(void)
     UtRegisterTest("AppLayerProtoDetectTest17", AppLayerProtoDetectTest17);
     UtRegisterTest("AppLayerProtoDetectTest18", AppLayerProtoDetectTest18);
     UtRegisterTest("AppLayerProtoDetectTest19", AppLayerProtoDetectTest19);
-    UtRegisterTest("AppLayerProtoDetectTest20", AppLayerProtoDetectTest20);
 
     SCReturn;
 }
diff --git a/src/app-layer.c b/src/app-layer.c
index 8cbb86838d..2bb7dc7d8b 100644
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -607,8 +607,6 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                                     flags, data, data_len);
             PACKET_PROFILING_APP_END(app_tctx, f->alproto);
-        } else {
-            SCLogDebug(" smsg not start, but no l7 data? Weird");
         }
     }
 
diff --git a/src/decode.h b/src/decode.h
index d732ce9038..16ab1a43e8 100644
--- a/src/decode.h
+++ b/src/decode.h
@@ -1108,6 +1108,8 @@ int DecoderParseDataFromFileSerie(char *fileprefix, DecoderFunc Decoder);
 
 #define PKT_REBUILT_FRAGMENT            (1<<25)     /**< Packet is rebuilt from
                                                      * fragments. */
+#define PKT_DETECT_HAS_STREAMDATA       (1<<26)     /**< Set by Detect() if raw stream data is available. */
+
 
 /** \brief return 1 if the packet is a pseudo packet */
 #define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END)
diff --git a/src/detect-engine-alert.c b/src/detect-engine-alert.c
index 5187bf9f3c..794331d117 100644
--- a/src/detect-engine-alert.c
+++ b/src/detect-engine-alert.c
@@ -184,7 +184,6 @@ int PacketAlertRemove(Packet *p, uint16_t pos)
  *  \param s the signature that matched
  *  \param p packet
  *  \param flags alert flags
- *  \param alert_msg ptr to StreamMsg object that the signature matched on
  */
 int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, const Signature *s,
         Packet *p, uint64_t tx_id, uint8_t flags)
diff --git a/src/detect-engine-payload.c b/src/detect-engine-payload.c
index ce65a08ba4..8e94dc2196 100644
--- a/src/detect-engine-payload.c
+++ b/src/detect-engine-payload.c
@@ -35,6 +35,7 @@
 #include "detect-engine-prefilter.h"
 
 #include "stream.h"
+#include "stream-tcp.h"
 
 #include "util-debug.h"
 #include "util-print.h"
@@ -45,27 +46,39 @@
 
 #include "util-mpm-ac.h"
 
+struct StreamMpmData {
+    DetectEngineThreadCtx *det_ctx;
+    const MpmCtx *mpm_ctx;
+};
+
+static int StreamMpmFunc(void *cb_data, const uint8_t *data, const uint32_t data_len)
+{
+    struct StreamMpmData *smd = cb_data;
+    if (data_len >= smd->mpm_ctx->minlen) {
+        (void)mpm_table[smd->mpm_ctx->mpm_type].Search(smd->mpm_ctx,
+                &smd->det_ctx->mtcs, &smd->det_ctx->pmq,
+                data, data_len);
+    }
+    return 0;
+}
+
 static void PrefilterPktStream(DetectEngineThreadCtx *det_ctx,
         Packet *p, const void *pectx)
 {
     SCEnter();
 
     const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    const StreamMsg *smsg = det_ctx->smsg;
-
-    /* for established packets inspect any smsg we may have queued up */
-    if (p->flowflags & FLOW_PKT_ESTABLISHED) {
-        SCLogDebug("p->flowflags & FLOW_PKT_ESTABLISHED");
-
-        for ( ; smsg != NULL; smsg = smsg->next) {
-            if (smsg->data_len >= mpm_ctx->minlen) {
-                (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                        &det_ctx->mtcs, &det_ctx->pmq,
-                        smsg->data, smsg->data_len);
-            }
-        }
+
+    /* for established packets inspect any stream we may have queued up */
+    if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
+        struct StreamMpmData stream_mpm_data = { det_ctx, mpm_ctx };
+        StreamReassembleRaw(p->flow->protoctx, p,
+                StreamMpmFunc, &stream_mpm_data,
+                &det_ctx->raw_stream_progress);
+        SCLogDebug("det_ctx->raw_stream_progress %"PRIu64,
+                det_ctx->raw_stream_progress);
     } else {
-        SCLogDebug("NOT p->flowflags & FLOW_PKT_ESTABLISHED");
+        SCLogDebug("NOT p->flags & PKT_DETECT_HAS_STREAMDATA");
     }
 
     /* packets that have not been added to the stream will be inspected
@@ -152,48 +165,61 @@ int DetectEngineInspectPacketPayload(DetectEngineCtx *de_ctx,
     SCReturnInt(0);
 }
 
+struct StreamContentInspectData {
+    DetectEngineCtx *de_ctx;
+    DetectEngineThreadCtx *det_ctx;
+    const Signature *s;
+    Flow *f;
+};
+
+static int StreamContentInspectFunc(void *cb_data, const uint8_t *data, const uint32_t data_len)
+{
+    SCEnter();
+    int r = 0;
+    struct StreamContentInspectData *smd = cb_data;
+
+    smd->det_ctx->buffer_offset = 0;
+    smd->det_ctx->discontinue_matching = 0;
+    smd->det_ctx->inspection_recursion_counter = 0;
+
+    r = DetectEngineContentInspection(smd->de_ctx, smd->det_ctx,
+            smd->s, smd->s->sm_arrays[DETECT_SM_LIST_PMATCH],
+            smd->f, (uint8_t *)data, data_len, 0,
+            DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM, NULL);
+    if (r == 1) {
+        SCReturnInt(1);
+    }
+
+    SCReturnInt(0);
+}
+
 /**
- *  \brief Do the content inspection & validation for a signature for a stream chunk
+ *  \brief Do the content inspection & validation for a signature
+ *         on the raw stream
  *
  *  \param de_ctx Detection engine context
  *  \param det_ctx Detection engine thread context
  *  \param s Signature to inspect
  *  \param f flow (for pcre flowvar storage)
- *  \param payload ptr to the payload to inspect
- *  \param payload_len length of the payload
  *
  *  \retval 0 no match
  *  \retval 1 match
- *
- *  \todo we might also pass the packet to this function for the pktvar
- *        storage. Only, would that be right? We're not inspecting data
- *        from the current packet here.
  */
 int DetectEngineInspectStreamPayload(DetectEngineCtx *de_ctx,
-        DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f,
-        uint8_t *payload, uint32_t payload_len)
+        DetectEngineThreadCtx *det_ctx, const Signature *s,
+        Flow *f, Packet *p)
 {
     SCEnter();
-    int r = 0;
-
-    if (s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL) {
-        SCReturnInt(0);
-    }
 
-    det_ctx->buffer_offset = 0;
-    det_ctx->discontinue_matching = 0;
-    det_ctx->inspection_recursion_counter = 0;
-
-    r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_arrays[DETECT_SM_LIST_PMATCH],
-                                      f, payload, payload_len, 0,
-                                      DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM, NULL);
-    if (r == 1) {
-        SCReturnInt(1);
-    }
-
-    SCReturnInt(0);
+    uint64_t unused;
+    struct StreamContentInspectData inspect_data = { de_ctx, det_ctx, s, f };
+    int r = StreamReassembleRaw(f->protoctx, p,
+            StreamContentInspectFunc, &inspect_data,
+            &unused);
+    return r;
 }
 
+
 #ifdef UNITTESTS
 
 /** \test Not the first but the second occurence of "abc" should be used
diff --git a/src/detect-engine-payload.h b/src/detect-engine-payload.h
index 205f8523f4..2e243ecf63 100644
--- a/src/detect-engine-payload.h
+++ b/src/detect-engine-payload.h
@@ -31,7 +31,7 @@ int DetectEngineInspectPacketPayload(DetectEngineCtx *,
         DetectEngineThreadCtx *, const Signature *, Flow *, Packet *);
 int DetectEngineInspectStreamPayload(DetectEngineCtx *,
         DetectEngineThreadCtx *, const Signature *, Flow *,
-        uint8_t *, uint32_t);
+        Packet *);
 
 void PayloadRegisterTests(void);
 
diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c
index 1bb2dc6495..5a1580fc05 100644
--- a/src/detect-engine-prefilter.c
+++ b/src/detect-engine-prefilter.c
@@ -141,7 +141,7 @@ static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx,
 }
 
 void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
-        Packet *p, const uint8_t flags, int has_state)
+        Packet *p, const uint8_t flags, const bool has_state)
 {
     SCEnter();
 
@@ -165,8 +165,9 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
 
     /* run payload inspecting engines */
     if (sgh->payload_engines &&
-        (p->payload_len > 0 || det_ctx->smsg != NULL) &&
-        !(p->flags & PKT_NOPAYLOAD_INSPECTION)) {
+        (p->payload_len || (p->flags & PKT_DETECT_HAS_STREAMDATA)) &&
+        !(p->flags & PKT_NOPAYLOAD_INSPECTION))
+    {
         PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_PAYLOAD);
         PrefilterEngine *engine = sgh->payload_engines;
         while (1) {
diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h
index b058cf738e..2c49136d9a 100644
--- a/src/detect-engine-prefilter.h
+++ b/src/detect-engine-prefilter.h
@@ -25,7 +25,7 @@
 #define __DETECT_ENGINE_PREFILTER_H__
 
 void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p,
-        const uint8_t flags, int has_state);
+        const uint8_t flags, const bool has_state);
 
 int PrefilterAppendEngine(SigGroupHead *sgh,
         void (*Prefilter)(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx),
diff --git a/src/detect-ssl-version.c b/src/detect-ssl-version.c
index 82ac3632eb..b6d9c39701 100644
--- a/src/detect-ssl-version.c
+++ b/src/detect-ssl-version.c
@@ -549,104 +549,6 @@ static int DetectSslVersionTestDetect02(void)
     PASS;
 }
 
-static int DetectSslVersionTestDetect03(void)
-{
-    DetectEngineCtx *de_ctx = NULL;
-    Flow f;
-    uint8_t sslbuf1[] = { 0x16 };
-    uint32_t ssllen1 = sizeof(sslbuf1);
-    uint8_t sslbuf2[] = { 0x03 };
-    uint32_t ssllen2 = sizeof(sslbuf2);
-    uint8_t sslbuf3[] = { 0x01 };
-    uint32_t ssllen3 = sizeof(sslbuf3);
-    uint8_t sslbuf4[] = { 0x01, 0x00, 0x00, 0xad, 0x03, 0x02 };
-    uint32_t ssllen4 = sizeof(sslbuf4);
-    TcpSession ssn;
-    Packet *p = NULL;
-    Signature *s = NULL;
-    ThreadVars th_v;
-    DetectEngineThreadCtx *det_ctx = NULL;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&th_v, 0, sizeof(th_v));
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
-    p->tcph->th_seq = htonl(1000);
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    p->flow = &f;
-    p->flowflags |= FLOW_PKT_TOSERVER;
-    p->flowflags |= FLOW_PKT_ESTABLISHED;
-    p->flags |= PKT_HAS_FLOW | PKT_STREAM_EST;
-    f.alproto = ALPROTO_TLS;
-    f.proto = p->proto;
-
-    StreamTcpInitConfig(TRUE);
-
-    StreamMsg *stream_msg = StreamMsgGetFromPool();
-    FAIL_IF_NULL(stream_msg);
-
-    memcpy(stream_msg->data, sslbuf4, ssllen4);
-    stream_msg->data_len = ssllen4;
-
-    ssn.toserver_smsg_head = stream_msg;
-    ssn.toserver_smsg_tail = stream_msg;
-
-    de_ctx = DetectEngineCtxInit();
-    FAIL_IF_NULL(de_ctx);
-
-    de_ctx->flags |= DE_QUIET;
-
-    s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"TLS\"; ssl_version:tls1.0; content:\"|01 00 00 AD|\"; sid:1;)");
-    FAIL_IF_NULL(s);
-
-    SigGroupBuild(de_ctx);
-    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
-
-    FLOWLOCK_WRLOCK(&f);
-    int r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS,
-                                STREAM_TOSERVER, sslbuf1, ssllen1);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            sslbuf2, ssllen2);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            sslbuf3, ssllen3);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            sslbuf4, ssllen4);
-    FAIL_IF(r != 0);
-    FLOWLOCK_UNLOCK(&f);
-
-    SSLState *app_state = f.alstate;
-    FAIL_IF_NULL(app_state);
-
-    FAIL_IF(app_state->client_connp.content_type != 0x16);
-
-    FAIL_IF(app_state->client_connp.version != TLS_VERSION_10);
-
-    /* do detect */
-    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
-
-    FAIL_IF_NOT(PacketAlertCheck(p, 1));
-
-    AppLayerParserThreadCtxFree(alp_tctx);
-    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
-    DetectEngineCtxFree(de_ctx);
-
-    StreamTcpFreeConfig(TRUE);
-    FLOW_DESTROY(&f);
-    UTHFreePackets(&p, 1);
-
-    PASS;
-}
-
 #endif /* UNITTESTS */
 
 /**
@@ -662,8 +564,6 @@ static void DetectSslVersionRegisterTests(void)
                    DetectSslVersionTestDetect01);
     UtRegisterTest("DetectSslVersionTestDetect02",
                    DetectSslVersionTestDetect02);
-    UtRegisterTest("DetectSslVersionTestDetect03",
-                   DetectSslVersionTestDetect03);
 #endif /* UNITTESTS */
 
     return;
diff --git a/src/detect-tls-version.c b/src/detect-tls-version.c
index 1f027b8247..8895e4483e 100644
--- a/src/detect-tls-version.c
+++ b/src/detect-tls-version.c
@@ -480,105 +480,6 @@ static int DetectTlsVersionTestDetect02(void)
 
     PASS;
 }
-
-static int DetectTlsVersionTestDetect03(void)
-{
-    DetectEngineCtx *de_ctx = NULL;
-    Flow f;
-    uint8_t tlsbuf1[] = { 0x16 };
-    uint32_t tlslen1 = sizeof(tlsbuf1);
-    uint8_t tlsbuf2[] = { 0x03 };
-    uint32_t tlslen2 = sizeof(tlsbuf2);
-    uint8_t tlsbuf3[] = { 0x01 };
-    uint32_t tlslen3 = sizeof(tlsbuf3);
-    uint8_t tlsbuf4[] = { 0x01, 0x00, 0x00, 0xad, 0x03, 0x02 };
-    uint32_t tlslen4 = sizeof(tlsbuf4);
-    TcpSession ssn;
-    Packet *p = NULL;
-    Signature *s = NULL;
-    ThreadVars th_v;
-    DetectEngineThreadCtx *det_ctx = NULL;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&th_v, 0, sizeof(th_v));
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
-    p->tcph->th_seq = htonl(1000);
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    p->flow = &f;
-    p->flowflags |= FLOW_PKT_TOSERVER;
-    p->flowflags |= FLOW_PKT_ESTABLISHED;
-    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
-    f.alproto = ALPROTO_TLS;
-    f.proto = p->proto;
-
-    StreamTcpInitConfig(TRUE);
-
-    StreamMsg *stream_msg = StreamMsgGetFromPool();
-    FAIL_IF_NULL(stream_msg);
-
-    memcpy(stream_msg->data, tlsbuf4, tlslen4);
-    stream_msg->data_len = tlslen4;
-
-    ssn.toserver_smsg_head = stream_msg;
-    ssn.toserver_smsg_tail = stream_msg;
-
-    de_ctx = DetectEngineCtxInit();
-    FAIL_IF_NULL(de_ctx);
-
-    de_ctx->flags |= DE_QUIET;
-
-    s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"TLS\"; tls.version:1.0; content:\"|01 00 00 AD|\"; sid:1;)");
-    FAIL_IF_NULL(s);
-
-    SigGroupBuild(de_ctx);
-    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
-
-    FLOWLOCK_WRLOCK(&f);
-    int r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS,
-                                STREAM_TOSERVER, tlsbuf1, tlslen1);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            tlsbuf2, tlslen2);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            tlsbuf3, tlslen3);
-    FAIL_IF(r != 0);
-
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
-                            tlsbuf4, tlslen4);
-    FAIL_IF(r != 0);
-    FLOWLOCK_UNLOCK(&f);
-
-    SSLState *ssl_state = f.alstate;
-    FAIL_IF_NULL(ssl_state);
-
-    FAIL_IF(ssl_state->client_connp.content_type != 0x16);
-
-    FAIL_IF(ssl_state->client_connp.version != TLS_VERSION_10);
-
-    /* do detect */
-    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
-
-    FAIL_IF_NOT(PacketAlertCheck(p, 1));
-
-    AppLayerParserThreadCtxFree(alp_tctx);
-    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
-    DetectEngineCtxFree(de_ctx);
-
-    StreamTcpFreeConfig(TRUE);
-    FLOW_DESTROY(&f);
-    UTHFreePackets(&p, 1);
-
-    PASS;
-}
-
 #endif /* UNITTESTS */
 
 /**
@@ -593,8 +494,6 @@ static void DetectTlsVersionRegisterTests(void)
                    DetectTlsVersionTestDetect01);
     UtRegisterTest("DetectTlsVersionTestDetect02",
                    DetectTlsVersionTestDetect02);
-    UtRegisterTest("DetectTlsVersionTestDetect03",
-                   DetectTlsVersionTestDetect03);
 #endif /* UNITTESTS */
 }
 
diff --git a/src/detect-uricontent.c b/src/detect-uricontent.c
index b541e08735..61227b2a04 100644
--- a/src/detect-uricontent.c
+++ b/src/detect-uricontent.c
@@ -998,19 +998,6 @@ static int DetectUriSigTest05(void)
     f.proto = p->proto;
 
     StreamTcpInitConfig(TRUE);
-
-    StreamMsg *stream_msg = StreamMsgGetFromPool();
-    if (stream_msg == NULL) {
-        printf("no stream_msg: ");
-        goto end;
-    }
-
-    memcpy(stream_msg->data, httpbuf1, httplen1);
-    stream_msg->data_len = httplen1;
-
-    ssn.toserver_smsg_head = stream_msg;
-    ssn.toserver_smsg_tail = stream_msg;
-
     de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL) {
         goto end;
@@ -1127,19 +1114,6 @@ static int DetectUriSigTest06(void)
     f.proto = p->proto;
 
     StreamTcpInitConfig(TRUE);
-
-    StreamMsg *stream_msg = StreamMsgGetFromPool();
-    if (stream_msg == NULL) {
-        printf("no stream_msg: ");
-        goto end;
-    }
-
-    memcpy(stream_msg->data, httpbuf1, httplen1);
-    stream_msg->data_len = httplen1;
-
-    ssn.toserver_smsg_head = stream_msg;
-    ssn.toserver_smsg_tail = stream_msg;
-
     de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL) {
         goto end;
diff --git a/src/detect.c b/src/detect.c
index 6edede5442..ca6cdb385d 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -237,7 +237,7 @@ void DetectExitPrintStats(ThreadVars *tv, void *data);
 
 void DbgPrintSigs(DetectEngineCtx *, SigGroupHead *);
 void DbgPrintSigs2(DetectEngineCtx *, SigGroupHead *);
-static void PacketCreateMask(Packet *, SignatureMask *, uint16_t, int, StreamMsg *, int);
+static void PacketCreateMask(Packet *, SignatureMask *, AppProto, bool, int);
 
 /**
  *  \brief Create the path if default-rule-path was specified
@@ -639,103 +639,6 @@ SigGroupHead *SigMatchSignaturesGetSgh(DetectEngineCtx *de_ctx, DetectEngineThre
     SCReturnPtr(sgh, "SigGroupHead");
 }
 
-/** \brief Get the smsgs relevant to this packet
- *
- *  \param f LOCKED flow
- *  \param p packet
- *  \param flags stream flags
- */
-static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags)
-{
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    StreamMsg *smsg = NULL;
-
-    if (p->proto == IPPROTO_TCP && f->protoctx != NULL && (p->flags & PKT_STREAM_EST)) {
-        TcpSession *ssn = (TcpSession *)f->protoctx;
-
-        /* at stream eof, or in inline mode, inspect all smsg's */
-        if ((flags & STREAM_EOF) || StreamTcpInlineMode()) {
-            if (p->flowflags & FLOW_PKT_TOSERVER) {
-                smsg = ssn->toserver_smsg_head;
-                /* deref from the ssn */
-                ssn->toserver_smsg_head = NULL;
-                ssn->toserver_smsg_tail = NULL;
-
-                SCLogDebug("to_server smsg %p at stream eof", smsg);
-                if (smsg)
-                    SCLogDebug("to_server smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
-            } else {
-                smsg = ssn->toclient_smsg_head;
-                /* deref from the ssn */
-                ssn->toclient_smsg_head = NULL;
-                ssn->toclient_smsg_tail = NULL;
-
-                SCLogDebug("to_client smsg %p at stream eof", smsg);
-                if (smsg)
-                    SCLogDebug("to_client smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
-            }
-        } else {
-            if (p->flowflags & FLOW_PKT_TOSERVER) {
-                StreamMsg *head = ssn->toserver_smsg_head;
-                if (unlikely(head == NULL)) {
-                    SCLogDebug("no smsgs in to_server direction");
-                    goto end;
-                }
-
-                /* if the smsg is bigger than the current packet, we will
-                 * process the smsg in a later run */
-                if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
-                    SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
-                            (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
-                    goto end;
-                }
-
-                smsg = head;
-                /* deref from the ssn */
-                ssn->toserver_smsg_head = NULL;
-                ssn->toserver_smsg_tail = NULL;
-
-                SCLogDebug("to_server smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
-            } else {
-                StreamMsg *head = ssn->toclient_smsg_head;
-                if (unlikely(head == NULL))
-                    goto end;
-
-                /* if the smsg is bigger than the current packet, we will
-                 * process the smsg in a later run */
-                if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
-                    SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
-                            (head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
-                    goto end;
-                }
-
-                smsg = head;
-                /* deref from the ssn */
-                ssn->toclient_smsg_head = NULL;
-                ssn->toclient_smsg_tail = NULL;
-
-                SCLogDebug("to_client smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
-            }
-        }
-    }
-
-end:
-#ifdef DEBUG
-    if (SCLogDebugEnabled()) {
-        StreamMsg *m = smsg;
-        while(m) {
-            SCLogDebug("m %p size %u, SEQ %u", m, m->data_len, m->seq);
-            PrintRawDataFp(stdout, m->data, m->data_len);
-            m = m->next;
-        }
-    }
-#endif
-    SCReturnPtr(smsg, "StreamMsg");
-}
-
 static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
                                             DetectEngineThreadCtx *det_ctx)
 {
@@ -863,17 +766,6 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
 #define SMS_USE_FLOW_SGH        0x01
 #define SMS_USED_PM             0x02
 
-#ifdef DEBUG
-static void DebugInspectIds(Packet *p, Flow *f, StreamMsg *smsg)
-{
-    SCLogDebug("pcap_cnt %02"PRIu64", %s, %12s, smsg %s",
-               p->pcap_cnt, p->flowflags & FLOW_PKT_TOSERVER ? "toserver" : "toclient",
-               p->flags & PKT_STREAM_EST ? "established" : "stateless",
-               smsg ? "yes" : "no");
-    AppLayerParserStatePrintDetails(f->alparser);
-}
-#endif
-
 static inline void
 DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask)
 {
@@ -1000,7 +892,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     int state_alert = 0;
     int alerts = 0;
     int app_decoder_events = 0;
-    int has_state = 0;          /* do we have an alstate to work with? */
+    bool has_state = false;     /* do we have an alstate to work with? */
 
     SCEnter();
 
@@ -1009,8 +901,8 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     det_ctx->ticker++;
     p->alerts.cnt = 0;
     det_ctx->filestore_cnt = 0;
-    det_ctx->smsg = NULL;
     det_ctx->base64_decoded_len = 0;
+    det_ctx->raw_stream_progress = 0;
 
     /* No need to perform any detection on this packet, if the the given flag is set.*/
     if (p->flags & PKT_NOPACKET_INSPECTION) {
@@ -1083,17 +975,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
                     sms_runflags |= SMS_USE_FLOW_SGH;
                 }
                 PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
-
-                det_ctx->smsg = SigMatchSignaturesGetSmsg(pflow, p, flow_flags);
-#if 0
-                StreamMsg *tmpsmsg = smsg;
-                while (tmpsmsg) {
-                    printf("detect ---start---:\n");
-                    PrintRawDataFp(stdout,tmpsmsg->data.data,tmpsmsg->data.data_len);
-                    printf("detect ---end---:\n");
-                    tmpsmsg = tmpsmsg->next;
-                }
-#endif
             }
 
             /* Retrieve the app layer state and protocol and the tcp reassembled
@@ -1106,6 +987,10 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
                 flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
                 has_state = (FlowGetAppState(pflow) != NULL);
                 alproto = FlowGetAppProtocol(pflow);
+                if (p->proto == IPPROTO_TCP && pflow->protoctx &&
+                    StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
+                    p->flags |= PKT_DETECT_HAS_STREAMDATA;
+                }
                 SCLogDebug("alstate %s, alproto %u", has_state ? "true" : "false", alproto);
             } else {
                 SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
@@ -1151,11 +1036,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
             PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
         }
 
-#ifdef DEBUG
-        if (pflow) {
-            DebugInspectIds(p, pflow, det_ctx->smsg);
-        }
-#endif
     } else { /* p->flags & PKT_HAS_FLOW */
         /* no flow */
 
@@ -1194,8 +1074,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
     /* create our prefilter mask */
     SignatureMask mask = 0;
-    PacketCreateMask(p, &mask, alproto, has_state, det_ctx->smsg,
-            app_decoder_events);
+    PacketCreateMask(p, &mask, alproto, has_state, app_decoder_events);
 
     /* build and prefilter non_pf list against the mask of the packet */
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
@@ -1382,33 +1261,25 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
             /* if we have stream msgs, inspect against those first,
              * but not for a "dsize" signature */
             if (sflags & SIG_FLAG_REQUIRE_STREAM) {
-                char pmatch = 0;
-                if (det_ctx->smsg != NULL) {
-                    StreamMsg *smsg_inspect = det_ctx->smsg;
-                    for ( ; smsg_inspect != NULL; smsg_inspect = smsg_inspect->next) {
-                        if (DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, pflow, smsg_inspect->data, smsg_inspect->data_len) == 1) {
-                            SCLogDebug("match in smsg %p", smsg_inspect);
-                            pmatch = 1;
-                            det_ctx->flags |= DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH;
-                            /* Tell the engine that this reassembled stream can drop the
-                             * rest of the pkts with no further inspection */
-                            if (s->action & ACTION_DROP)
-                              alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
-
-                            alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
-                            break;
-                        }
+                int pmatch = 0;
+                if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
+                    pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, pflow, p);
+                    if (pmatch) {
+                        det_ctx->flags |= DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH;
+                        /* Tell the engine that this reassembled stream can drop the
+                         * rest of the pkts with no further inspection */
+                        if (s->action & ACTION_DROP)
+                            alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
+
+                        alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
                     }
-
-                } /* if (smsg != NULL) */
-
+                }
                 /* no match? then inspect packet payload */
                 if (pmatch == 0) {
                     SCLogDebug("no match in smsg, fall back to packet payload");
 
-                    if (!(sflags & SIG_FLAG_REQUIRE_PACKET)) {
-                        if (p->flags & PKT_STREAM_ADD)
-                            goto next;
+                    if (!(sflags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
+                        goto next;
                     }
 
                     if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, pflow, p) != 1) {
@@ -1448,7 +1319,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
         /* consider stateful sig matches */
         if (sflags & SIG_FLAG_STATE_MATCH) {
-            if (has_state == 0) {
+            if (has_state == false) {
                 SCLogDebug("state matches but no state, we can't match");
                 goto next;
             }
@@ -1542,10 +1413,13 @@ end:
             DetectPostInspectFirstSGH(p, pflow, det_ctx->sgh);
         }
 
-        /* if we had no alerts that involved the smsgs,
-         * we can get rid of them now. */
-        StreamMsgReturnListToPool(det_ctx->smsg);
-        det_ctx->smsg = NULL;
+        /* update inspected tracker for raw reassembly */
+        if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
+            det_ctx->raw_stream_progress > 0)
+        {
+            StreamReassembleRawUpdateProgress(pflow->protoctx, p,
+                    det_ctx->raw_stream_progress);
+        }
     }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_CLEANUP);
 
@@ -2062,13 +1936,15 @@ deonly:
  * SIG_MASK_REQUIRE_HTTP_STATE, SIG_MASK_REQUIRE_DCE_STATE
  */
 static void
-PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, int has_state, StreamMsg *smsg,
-        int app_decoder_events)
+PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
+        bool has_state, int app_decoder_events)
 {
-    /* no payload inspect flag doesn't apply to smsg */
-    if (smsg != NULL || (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0)) {
+    if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
         SCLogDebug("packet has payload");
         (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
+    } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
+        SCLogDebug("stream data available");
+        (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
     } else {
         SCLogDebug("packet has no payload");
         (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
diff --git a/src/detect.h b/src/detect.h
index bda17fb4e8..1a3286d66e 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -791,6 +791,8 @@ typedef struct DetectEngineThreadCtx_ {
 
     /* detection engine variables */
 
+    uint64_t raw_stream_progress;
+
     /** offset into the payload of the last match by:
      *  content, pcre, etc */
     uint32_t buffer_offset;
@@ -864,8 +866,6 @@ typedef struct DetectEngineThreadCtx_ {
     MpmThreadCtx mtcs;  /**< thread ctx for stream mpm */
     PrefilterRuleStore pmq;
 
-    StreamMsg *smsg;
-
     /** SPM thread context used for scanning. This has been cloned from the
      * prototype held by DetectEngineCtx. */
     SpmThreadCtx *spm_thread_ctx;
diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c
index 524a0a4e16..89e3ffb1ae 100644
--- a/src/stream-tcp-list.c
+++ b/src/stream-tcp-list.c
@@ -584,18 +584,6 @@ int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
 
 static inline int SegmentInUse(TcpSession *ssn, TcpStream *stream, TcpSegment *seg)
 {
-    if (stream == &ssn->client && ssn->toserver_smsg_head != NULL) {
-        /* not (seg is entirely before first smsg, skip) */
-        if (!(SEQ_LEQ(seg->seq + TCP_SEG_LEN(seg), ssn->toserver_smsg_head->seq))) {
-            SCReturnInt(1);
-        }
-    } else if (stream == &ssn->server && ssn->toclient_smsg_head != NULL) {
-        /* not (seg is entirely before first smsg, skip) */
-        if (!(SEQ_LEQ(seg->seq + TCP_SEG_LEN(seg), ssn->toclient_smsg_head->seq))) {
-            SCReturnInt(1);
-        }
-    }
-
     /* if proto detect isn't done, we're not returning */
     if (!(stream->flags & STREAMTCP_STREAM_FLAG_GAP)) {
         if (!(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream))) {
@@ -610,9 +598,6 @@ static inline int SegmentInUse(TcpSession *ssn, TcpStream *stream, TcpSegment *s
 /** \internal
  *  \brief check if we can remove a segment from our segment list
  *
- *  If a segment is entirely before the oldest smsg, we can discard it. Otherwise
- *  we keep it around to be able to log it.
- *
  *  \retval 1 yes
  *  \retval 0 no
  */
diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h
index 7e78c7fbad..f51e8ac8ff 100644
--- a/src/stream-tcp-private.h
+++ b/src/stream-tcp-private.h
@@ -232,11 +232,6 @@ typedef struct TcpSession_ {
     uint32_t reassembly_depth;      /**< reassembly depth for the stream */
     TcpStream server;
     TcpStream client;
-    struct StreamMsg_ *toserver_smsg_head;  /**< list of stream msgs (for detection inspection) */
-    struct StreamMsg_ *toserver_smsg_tail;  /**< list of stream msgs (for detection inspection) */
-    struct StreamMsg_ *toclient_smsg_head;  /**< list of stream msgs (for detection inspection) */
-    struct StreamMsg_ *toclient_smsg_tail;  /**< list of stream msgs (for detection inspection) */
-
     TcpStateQueue *queue;                   /**< list of SYN/ACK candidates */
 } TcpSession;
 
diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 8f471a5a00..48587b7efc 100644
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -363,21 +363,14 @@ int StreamTcpReassemblyConfig(char quiet)
         SCLogInfo("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
     stream_config.prealloc_segments = segment_prealloc;
 
-    uint32_t stream_chunk_prealloc = 250;
-    ConfNode *chunk = ConfGetNode("stream.reassembly.chunk-prealloc");
-    if (chunk) {
-        uint32_t prealloc = 0;
-        if (ByteExtractStringUint32(&prealloc, 10, strlen(chunk->val), chunk->val) == -1)
-        {
-            SCLogError(SC_ERR_INVALID_ARGUMENT, "chunk-prealloc of "
-                    "%s is invalid", chunk->val);
-            return -1;
-        }
-        stream_chunk_prealloc = prealloc;
+    int overlap_diff_data = 0;
+    ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
+    if (overlap_diff_data) {
+        StreamTcpReassembleConfigEnableOverlapCheck();
+    }
+    if (StreamTcpInlineMode() == TRUE) {
+        StreamTcpReassembleConfigEnableOverlapCheck();
     }
-    if (!quiet)
-        SCLogConfig("stream.reassembly \"chunk-prealloc\": %u", stream_chunk_prealloc);
-    StreamMsgQueuesInit(stream_chunk_prealloc);
 
     stream_config.sbcnf.flags = STREAMING_BUFFER_NOFLAGS;
     stream_config.sbcnf.buf_size = 2048;
@@ -407,8 +400,6 @@ int StreamTcpReassembleInit(char quiet)
 
 void StreamTcpReassembleFree(char quiet)
 {
-    StreamMsgQueuesDeinit(quiet);
-
     SCMutexLock(&segment_thread_pool_mutex);
     if (segment_thread_pool != NULL) {
         PoolThreadFree(segment_thread_pool);
@@ -584,56 +575,6 @@ static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream
     SCReturnUInt(0);
 }
 
-static void StreamTcpStoreStreamChunk(TcpSession *ssn, StreamMsg *smsg, const Packet *p, int streaminline)
-{
-    uint8_t direction = 0;
-
-    if ((!streaminline && (p->flowflags & FLOW_PKT_TOSERVER)) ||
-        ( streaminline && (p->flowflags & FLOW_PKT_TOCLIENT)))
-    {
-        direction = STREAM_TOCLIENT;
-        SCLogDebug("stream chunk is to_client");
-    } else {
-        direction = STREAM_TOSERVER;
-        SCLogDebug("stream chunk is to_server");
-    }
-
-    /* store the smsg in the tcp stream */
-    if (direction == STREAM_TOSERVER) {
-        SCLogDebug("storing smsg in the to_server");
-
-        /* put the smsg in the stream list */
-        if (ssn->toserver_smsg_head == NULL) {
-            ssn->toserver_smsg_head = smsg;
-            ssn->toserver_smsg_tail = smsg;
-            smsg->next = NULL;
-            smsg->prev = NULL;
-        } else {
-            StreamMsg *cur = ssn->toserver_smsg_tail;
-            cur->next = smsg;
-            smsg->prev = cur;
-            smsg->next = NULL;
-            ssn->toserver_smsg_tail = smsg;
-        }
-    } else {
-        SCLogDebug("storing smsg in the to_client");
-
-        /* put the smsg in the stream list */
-        if (ssn->toclient_smsg_head == NULL) {
-            ssn->toclient_smsg_head = smsg;
-            ssn->toclient_smsg_tail = smsg;
-            smsg->next = NULL;
-            smsg->prev = NULL;
-        } else {
-            StreamMsg *cur = ssn->toclient_smsg_tail;
-            cur->next = smsg;
-            smsg->prev = cur;
-            smsg->next = NULL;
-            ssn->toclient_smsg_tail = smsg;
-        }
-    }
-}
-
 /**
  *  \brief Insert a packets TCP data into the stream reassembly engine.
  *
@@ -790,47 +731,46 @@ static int StreamTcpReassembleRawCheckLimit(TcpSession *ssn,
         SCReturnInt(1);
 
     /* check if we have enough data to do raw reassembly */
-    if (p->flowflags & FLOW_PKT_TOCLIENT) {
-        SCLogDebug("StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER) %"PRIu32,
-                StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER));
-
-        uint32_t delta = stream->last_ack - stream->base_seq;
-        /* get max absolute offset */
-        uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
-
-        int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
-
-        if ((int64_t)StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER) >
-                diff) {
-            SCLogDebug("toserver min chunk len not yet reached: "
-                    "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
-                    "%"PRIu32"", stream->last_ack, stream->base_seq,
-                    (stream->last_ack - stream->base_seq),
-                    StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER));
-            SCReturnInt(0);
+    if (PKT_IS_TOSERVER(p)) {
+        if (STREAM_LASTACK_GT_BASESEQ(stream)) {
+            uint32_t delta = stream->last_ack - stream->base_seq;
+            /* get max absolute offset */
+            uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
+
+            int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
+            if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
+                SCReturnInt(1);
+            } else {
+                SCLogDebug("toserver min chunk len not yet reached: "
+                        "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
+                        "%"PRIu32"", stream->last_ack, stream->base_seq,
+                        (stream->last_ack - stream->base_seq),
+                        stream_config.reassembly_toserver_chunk_size);
+                SCReturnInt(0);
+            }
         }
     } else {
-        SCLogDebug("StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT) %"PRIu32,
-                StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT));
-
-        uint32_t delta = stream->last_ack - stream->base_seq;
-        /* get max absolute offset */
-        uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
+        if (STREAM_LASTACK_GT_BASESEQ(stream)) {
+            uint32_t delta = stream->last_ack - stream->base_seq;
+            /* get max absolute offset */
+            uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
 
-        int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
+            int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
 
-        if ((int64_t)StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT) >
-                diff) {
-            SCLogDebug("toclient min chunk len not yet reached: "
-                    "last_ack %"PRIu32", base_seq %"PRIu32",  %"PRIu32" < "
-                    "%"PRIu32"", stream->last_ack, stream->base_seq,
-                    (stream->last_ack - stream->base_seq),
-                    StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT));
-            SCReturnInt(0);
+            if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
+                SCReturnInt(1);
+            } else {
+                SCLogDebug("toclient min chunk len not yet reached: "
+                        "last_ack %"PRIu32", base_seq %"PRIu32",  %"PRIu32" < "
+                        "%"PRIu32"", stream->last_ack, stream->base_seq,
+                        (stream->last_ack - stream->base_seq),
+                        stream_config.reassembly_toclient_chunk_size);
+                SCReturnInt(0);
+            }
         }
     }
 
-    SCReturnInt(1);
+    SCReturnInt(0);
 }
 
 /**
@@ -839,20 +779,17 @@ static int StreamTcpReassembleRawCheckLimit(TcpSession *ssn,
 int StreamNeedsReassembly(TcpSession *ssn, int direction)
 {
     TcpStream *stream = NULL;
-    StreamMsg *head = NULL;
 #ifdef DEBUG
     char *dirstr = NULL;
 #endif
     /* TODO use STREAM_TOCLIENT/STREAM_TOSERVER */
     if (direction == 0) {
         stream = &ssn->client;
-        head = ssn->toserver_smsg_head;
 #ifdef DEBUG
         dirstr = "client";
 #endif
     } else {
         stream = &ssn->server;
-        head = ssn->toclient_smsg_head;
 #ifdef DEBUG
         dirstr = "server";
 #endif
@@ -900,11 +837,6 @@ int StreamNeedsReassembly(TcpSession *ssn, int direction)
         SCLogDebug("%s: no list", dirstr);
     }
 
-    if (head != NULL) {
-        SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
-        return STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION;
-    }
-
     SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
     return STREAM_HAS_UNPROCESSED_SEGMENTS_NONE;
 }
@@ -1225,12 +1157,97 @@ static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_
     return 0;
 }
 
-/** \internal
- *  \brief based on the data in the streaming buffer setup StreamMsgs
+/** \brief does the stream engine have data to inspect?
+ *
+ *  Returns true if there is data to inspect. In IDS case this is
+ *  about ACK'd data in the packet's direction.
+ *
+ *  In the IPS case this is about the packet itself.
+ */
+bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
+{
+    TcpStream *stream;
+    if (PKT_IS_TOSERVER(p)) {
+        stream = &ssn->client;
+    } else {
+        stream = &ssn->server;
+    }
+
+    uint64_t progress = STREAM_RAW_PROGRESS(stream);
+    if (StreamTcpInlineMode() == FALSE) {
+        if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
+            uint32_t delta = stream->last_ack - stream->base_seq;
+            /* get max absolute offset */
+            uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream) + delta;
+            if (last_ack_abs > progress) {
+                return true;
+            }
+        }
+    } else {
+        if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/** \brief Update stream after inspection has run
+ */
+void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
+{
+    TcpStream *stream;
+    if (PKT_IS_TOSERVER(p)) {
+        stream = &ssn->client;
+    } else {
+        stream = &ssn->server;
+    }
+
+    if (progress > STREAM_RAW_PROGRESS(stream)) {
+        uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
+        stream->raw_progress_rel += slide;
+    }
+
+    SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
+}
+
+/** \brief access 'raw' reassembly data.
+ *
+ *  Access data as tracked by 'raw' tracker. Data is made available to
+ *  callback that is passed to this function.
+ *
+ *  In the case of IDS the callback may be run multiple times if data
+ *  contains gaps. It will then be run for each block of data that is
+ *  continuous.
+ *
+ *  The callback should give on of 2 return values:
+ *  - 0 ok
+ *  - 1 done
+ *  The value 1 will break the loop if there is a block list that is
+ *  inspected.
+ *
+ *  This function will return the 'progress' value that has been
+ *  consumed until now.
+ *
+ *  \param ssn tcp session
+ *  \param p packet
+ *  \param Callback the function pointer to the callback function
+ *  \param cb_data callback data
+ *  \param[out] progress_out absolute progress value of the data this
+ *                           call handled.
  */
-static int ReassembleRaw(TcpSession *ssn, TcpStream *stream, Packet *p)
+int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
+                        StreamReassembleRawFunc Callback, void *cb_data,
+                        uint64_t *progress_out)
 {
     SCEnter();
+    int r = 0;
+
+    TcpStream *stream;
+    if (PKT_IS_TOSERVER(p)) {
+        stream = &ssn->client;
+    } else {
+        stream = &ssn->server;
+    }
 
     StreamingBufferBlock *iter = NULL;
     uint64_t progress = STREAM_RAW_PROGRESS(stream);
@@ -1309,9 +1326,9 @@ static int ReassembleRaw(TcpSession *ssn, TcpStream *stream, Packet *p)
         /* StreamTcpInlineMode() == TRUE */
         } else {
             if (progress + mydata_len > right_edge_abs) {
-                uint32_t delta = (progress + mydata_len) - right_edge_abs;
-                SCLogDebug("adjusting mydata_len %u to subtract %u", mydata_len, delta);
-                mydata_len -= delta;
+                uint32_t delta2 = (progress + mydata_len) - right_edge_abs;
+                SCLogDebug("adjusting mydata_len %u to subtract %u", mydata_len, delta2);
+                mydata_len -= delta2;
             }
         }
         if (mydata_len == 0)
@@ -1331,43 +1348,15 @@ static int ReassembleRaw(TcpSession *ssn, TcpStream *stream, Packet *p)
 #define GET_SEQ_FOR_PROGRESS(stream, progress) \
             (((progress) - STREAM_BASE_OFFSET((stream))) + (stream->base_seq))
 
-        /* we have data. Use it to setup StreamMsg(s) */
-        StreamMsg *smsg = NULL;
-        uint32_t data_offset = 0;
-        uint32_t data_left = mydata_len;
-        while (data_left) {
-            smsg = StreamMsgGetFromPool();
-            if (smsg == NULL)
-                break;
-
-            uint32_t copy_len = (data_left > smsg->data_size) ? smsg->data_size : data_left;
-            SCLogDebug("copy_len %u, data_left %u", copy_len, data_left);
-
-            memcpy(smsg->data, mydata + data_offset, copy_len);
-            smsg->data_len = copy_len;
-            smsg->seq = GET_SEQ_FOR_PROGRESS(stream, (mydata_offset + data_offset));
-            SCLogDebug("smsg %p seq %u", smsg, smsg->seq);
-
-            BUG_ON(copy_len > data_left);
-            data_left -= copy_len;
-            BUG_ON(data_left > mydata_len);
-            data_offset += copy_len;
-
-            SCLogDebug("smsg %p %u/%u", smsg, smsg->data_len, smsg->data_size);
-            //PrintRawDataFp(stdout, smsg->data, smsg->data_len);
-
-            StreamTcpStoreStreamChunk(ssn, smsg, p, StreamTcpInlineMode());
-        }
+        /* we have data. */
+        r = Callback(cb_data, mydata, mydata_len);
+        BUG_ON(r < 0);
 
         if (mydata_offset == progress) {
             SCLogDebug("raw progress %"PRIu64" increasing with data len %u to %"PRIu64,
                     progress, mydata_len, STREAM_RAW_PROGRESS(stream) + mydata_len);
 
-            //if (StreamTcpInlineMode() == TRUE) {
-            //progress = right_edge_abs;
-            //} else {
             progress += mydata_len;
-            //}
             SCLogDebug("raw progress now %"PRIu64, progress);
 
         /* data is beyond the progress we'd like, and before last ack. Gap. */
@@ -1382,56 +1371,12 @@ static int ReassembleRaw(TcpSession *ssn, TcpStream *stream, Packet *p)
                        "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
         }
 
-        if (iter == NULL)
+        if (iter == NULL || r == 1)
             break;
     }
 end:
-    if (progress > STREAM_RAW_PROGRESS(stream)) {
-        uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
-        stream->raw_progress_rel += slide;
-    }
-
-    SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
-    return 0;
-}
-
-/**
- *  \brief Update the stream reassembly upon receiving an ACK packet.
- *  \todo this function is too long, we need to break it up. It needs it BAD
- */
-int StreamTcpReassembleRaw (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
-        TcpSession *ssn, TcpStream *stream, Packet *p)
-{
-    SCEnter();
-    SCLogDebug("start p %p", p);
-
-    if (ssn->flags & STREAMTCP_FLAG_DISABLE_RAW)
-        SCReturnInt(0);
-
-    if (stream->seg_list == NULL) {
-        SCLogDebug("no segments in the list to reassemble");
-        SCReturnInt(0);
-    }
-
-#if 0
-    if (ssn->state <= TCP_ESTABLISHED &&
-        !StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream)) {
-        SCLogDebug("only starting raw reassembly after app layer protocol "
-                "detection has completed.");
-        SCReturnInt(0);
-    }
-#endif
-    /* check if we have enough data */
-    if (StreamTcpInlineMode() == FALSE &&
-            StreamTcpReassembleRawCheckLimit(ssn,stream,p) == 0)
-    {
-        SCLogDebug("not yet reassembling");
-        SCReturnInt(0);
-    }
-
-    /* take the data we have, and turn it into StreamMsgs */
-    ReassembleRaw(ssn, stream, p);
-    SCReturnInt(0);
+    *progress_out = progress;
+    return r;
 }
 
 /** \brief update app layer and raw reassembly
@@ -1449,8 +1394,6 @@ int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
     if (!(StreamTcpInlineMode())) {
         if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p) < 0)
             r = -1;
-        if (StreamTcpReassembleRaw(tv, ra_ctx, ssn, stream, p) < 0)
-            r = -1;
     }
 
     SCLogDebug("stream->seg_list %p", stream->seg_list);
@@ -1500,8 +1443,6 @@ int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
         int r = 0;
         if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p) < 0)
             r = -1;
-        if (StreamTcpReassembleRaw(tv, ra_ctx, ssn, stream, p) < 0)
-            r = -1;
 
         if (r < 0) {
             SCReturnInt(-1);
@@ -1525,7 +1466,6 @@ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx)
            segment request due to memcap limit */
         StatsIncr(tv, ra_ctx->counter_tcp_segment_memcap);
     } else {
-        seg->flags = stream_config.segment_init_flags;
         seg->next = NULL;
         seg->prev = NULL;
 
@@ -1567,272 +1507,6 @@ void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn)
     (stream)->isn = (setseq);               \
     (stream)->base_seq = (setseq) + 1
 
-static int UtTestSmsg(StreamMsg *smsg, const uint8_t *buf, uint32_t buf_len)
-{
-    if (smsg == NULL)
-        return 0;
-
-    if (smsg->data_len != buf_len) {
-        printf("Got: data_len %u, expected %u\n", smsg->data_len, buf_len);
-        PrintRawDataFp(stdout, smsg->data, smsg->data_len);
-        return 0;
-    }
-
-    if (!(memcmp(buf, smsg->data, buf_len) == 0)) {
-        printf("data is not what we expected:\nExpected:\n");
-        PrintRawDataFp(stdout, (uint8_t *)buf, buf_len);
-        printf("Got:\n");
-        PrintRawDataFp(stdout, smsg->data, smsg->data_len);
-        return 0;
-    }
-    return 1;
-}
-
-static uint32_t UtSsnSmsgCnt(TcpSession *ssn, uint8_t direction)
-{
-    uint32_t cnt = 0;
-    StreamMsg *smsg = (direction == STREAM_TOSERVER) ?
-                            ssn->toserver_smsg_head :
-                            ssn->toclient_smsg_head;
-    while (smsg) {
-        cnt++;
-        smsg = smsg->next;
-    }
-    return cnt;
-}
-
-#if 0
-/** \brief  The Function tests the reassembly engine working for different
- *          OSes supported. It includes all the OS cases and send
- *          crafted packets to test the reassembly.
- *
- *  \param  stream  The stream which will contain the reassembled segments
- */
-
-static int StreamTcpReassembleStreamTest(TcpStream *stream)
-{
-
-    TcpSession ssn;
-    Packet *p = PacketGetFromAlloc();
-    if (unlikely(p == NULL))
-        return 0;
-    Flow f;
-    uint8_t payload[4];
-    TCPHdr tcph;
-    TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(NULL);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-    PacketQueue pq;
-    memset(&pq,0,sizeof(PacketQueue));
-
-    memset(&ssn, 0, sizeof (TcpSession));
-    memset(&f, 0, sizeof (Flow));
-    memset(&tcph, 0, sizeof (TCPHdr));
-    ThreadVars tv;
-    memset(&tv, 0, sizeof (ThreadVars));
-    FLOW_INITIALIZE(&f);
-    f.protoctx = &ssn;
-    f.proto = IPPROTO_TCP;
-    p->src.family = AF_INET;
-    p->dst.family = AF_INET;
-    p->proto = IPPROTO_TCP;
-    p->flow = &f;
-    tcph.th_win = 5480;
-    tcph.th_flags = TH_PUSH | TH_ACK;
-    p->tcph = &tcph;
-    p->flowflags = FLOW_PKT_TOSERVER;
-    SET_ISN(&ssn.client, 10);
-    SET_ISN(&ssn.server, 10);
-
-    StreamTcpCreateTestPacket(payload, 0x41, 3, 4); /*AAA*/
-    p->tcph->th_seq = htonl(12);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x42, 2, 4); /*BB*/
-    p->tcph->th_seq = htonl(16);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 2;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x43, 3, 4); /*CCC*/
-    p->tcph->th_seq = htonl(18);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x44, 1, 4); /*D*/
-    p->tcph->th_seq = htonl(22);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x45, 2, 4); /*EE*/
-    p->tcph->th_seq = htonl(25);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 2;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x46, 3, 4); /*FFF*/
-    p->tcph->th_seq = htonl(27);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x47, 2, 4); /*GG*/
-    p->tcph->th_seq = htonl(30);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 2;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x48, 2, 4); /*HH*/
-    p->tcph->th_seq = htonl(32);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 2;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x49, 1, 4); /*I*/
-    p->tcph->th_seq = htonl(34);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4a, 4, 4); /*JJJJ*/
-    p->tcph->th_seq = htonl(13);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 4;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4b, 3, 4); /*KKK*/
-    p->tcph->th_seq = htonl(18);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4c, 3, 4); /*LLL*/
-    p->tcph->th_seq = htonl(21);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4d, 3, 4); /*MMM*/
-    p->tcph->th_seq = htonl(24);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 3;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4e, 1, 4); /*N*/
-    p->tcph->th_seq = htonl(28);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x4f, 1, 4); /*O*/
-    p->tcph->th_seq = htonl(31);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x50, 1, 4); /*P*/
-    p->tcph->th_seq = htonl(32);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x51, 2, 4); /*QQ*/
-    p->tcph->th_seq = htonl(34);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 2;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpCreateTestPacket(payload, 0x30, 1, 4); /*0*/
-    p->tcph->th_seq = htonl(11);
-    p->tcph->th_ack = htonl(31);
-    p->payload = payload;
-    p->payload_len = 1;
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, stream, p, &pq) == -1) {
-        SCFree(p);
-        return 0;
-    }
-
-    StreamTcpReassembleFreeThreadCtx(ra_ctx);
-
-    SCFree(p);
-    return 1;
-}
-#endif
-
 /** \brief  The Function to create the packet with given payload, which is used
  *          to test the reassembly of the engine.
  *
@@ -2064,11 +1738,6 @@ static int StreamTcpReassembleTest34(void)
 
     StreamTcpUTInit(&ra_ctx);
     StreamTcpUTSetupSession(&ssn);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     PacketQueue pq;
     memset(&pq,0,sizeof(PacketQueue));
     memset(&f, 0, sizeof (Flow));
@@ -2135,11 +1804,6 @@ static int StreamTcpReassembleTest37(void)
 
     StreamTcpUTInit(&ra_ctx);
     StreamTcpUTSetupSession(&ssn);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10);
-
     memset(&pq,0,sizeof(PacketQueue));
     memset(&f, 0, sizeof (Flow));
     memset(&tcph, 0, sizeof (TCPHdr));
@@ -2188,98 +1852,6 @@ static int StreamTcpReassembleTest37(void)
     PASS;
 }
 
-/**
- *  \test   Test to make sure we don't send the smsg from toclient to app layer
- *          until the app layer protocol has been detected and one smsg from
- *          toserver side has been sent to app layer.
- *
- *  \retval On success it returns 1 and on failure 0.
- */
-static int StreamTcpReassembleTest38 (void)
-{
-    Packet *p = PacketGetFromAlloc();
-    FAIL_IF(unlikely(p == NULL));
-    Flow f;
-    TCPHdr tcph;
-    Port sp;
-    Port dp;
-    struct in_addr in;
-    TcpSession ssn;
-    PacketQueue pq;
-    memset(&pq,0,sizeof(PacketQueue));
-    memset(&f, 0, sizeof (Flow));
-    memset(&tcph, 0, sizeof (TCPHdr));
-    ThreadVars tv;
-    memset(&tv, 0, sizeof (ThreadVars));
-    TcpReassemblyThreadCtx *ra_ctx = NULL;
-
-    StreamTcpUTInit(&ra_ctx);
-    StreamTcpUTSetupSession(&ssn);
-
-    uint8_t httpbuf2[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n";
-    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
-
-    uint8_t httpbuf1[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
-    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
-
-    FLOW_INITIALIZE(&f);
-    FAIL_IF(inet_pton(AF_INET, "1.2.3.4", &in) != 1);
-    f.src.addr_data32[0] = in.s_addr;
-    FAIL_IF(inet_pton(AF_INET, "1.2.3.5", &in) != 1);
-    f.dst.addr_data32[0] = in.s_addr;
-    sp = 200;
-    dp = 220;
-
-    SET_ISN(&ssn.server, 9);
-    ssn.server.last_ack = 60;
-    SET_ISN(&ssn.client, 9);
-    ssn.client.last_ack = 9;
-    f.alproto = ALPROTO_UNKNOWN;
-
-    f.flags |= FLOW_IPV4;
-    f.sp = sp;
-    f.dp = dp;
-    f.protoctx = &ssn;
-    f.proto = IPPROTO_TCP;
-    p->flow = &f;
-
-    tcph.th_win = htons(5480);
-    tcph.th_seq = htonl(10);
-    tcph.th_ack = htonl(20);
-    tcph.th_flags = TH_ACK|TH_PUSH;
-    p->tcph = &tcph;
-    p->flowflags = FLOW_PKT_TOSERVER;
-
-    p->payload = httpbuf2;
-    p->payload_len = httplen2;
-    ssn.state = TCP_ESTABLISHED;
-
-    TcpStream *s = NULL;
-    s = &ssn.server;
-
-    FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
-
-    /* Check if we have stream smsgs in queue */
-    FAIL_IF(UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) > 0);
-
-    p->flowflags = FLOW_PKT_TOCLIENT;
-    p->payload = httpbuf1;
-    p->payload_len = httplen1;
-    tcph.th_seq = htonl(10);
-    tcph.th_ack = htonl(55);
-    s = &ssn.client;
-
-    FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
-
-    /* Check if we have stream smsgs in queue */
-    FAIL_IF(UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1);
-
-    StreamTcpUTClearSession(&ssn);
-    StreamTcpUTDeinit(ra_ctx);
-    SCFree(p);
-    PASS;
-}
-
 /**
  *  \test   Test to make sure that we don't return the segments until the app
  *          layer proto has been detected and after that remove the processed
@@ -2336,8 +1908,6 @@ static int StreamTcpReassembleTest39 (void)
         FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT) || FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT) ||
         ssn->client.seg_list != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != 0) {
         printf("failure 1\n");
         goto end;
@@ -2361,8 +1931,6 @@ static int StreamTcpReassembleTest39 (void)
         FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT) || FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT) ||
         ssn->client.seg_list != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != 0) {
         printf("failure 2\n");
         goto end;
@@ -2387,8 +1955,6 @@ static int StreamTcpReassembleTest39 (void)
         FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT) || FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT) ||
         ssn->client.seg_list != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != 0) {
         printf("failure 3\n");
         goto end;
@@ -2415,8 +1981,6 @@ static int StreamTcpReassembleTest39 (void)
         ssn->client.seg_list == NULL ||
         ssn->client.seg_list->next != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != STREAM_TOSERVER) {
         printf("failure 4\n");
         goto end;
@@ -2443,8 +2007,6 @@ static int StreamTcpReassembleTest39 (void)
         ssn->client.seg_list == NULL ||
         ssn->client.seg_list->next != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != STREAM_TOSERVER) {
         printf("failure 5\n");
         goto end;
@@ -2483,8 +2045,6 @@ static int StreamTcpReassembleTest39 (void)
         ssn->client.seg_list->next == NULL ||
         ssn->client.seg_list->next->next != NULL ||
         ssn->server.seg_list != NULL ||
-        ssn->toserver_smsg_head != NULL ||
-        ssn->toclient_smsg_head != NULL ||
         ssn->data_first_seen_dir != STREAM_TOSERVER) {
         printf("failure 6\n");
         goto end;
@@ -2776,21 +2336,6 @@ static int StreamTcpReassembleTest39 (void)
         printf("failure 15\n");
         goto end;
     }
-
-    if (ssn->toserver_smsg_head == NULL ||
-        ssn->toserver_smsg_head->next == NULL ||
-        ssn->toserver_smsg_head->next->next != NULL ||
-        ssn->toclient_smsg_head == NULL ||
-        ssn->toclient_smsg_head->next != NULL) {
-        printf("failure 16\n");
-        goto end;
-    }
-
-    StreamMsgReturnListToPool(ssn->toserver_smsg_head);
-    ssn->toserver_smsg_head = ssn->toserver_smsg_tail = NULL;
-    StreamMsgReturnListToPool(ssn->toclient_smsg_head);
-    ssn->toclient_smsg_head = ssn->toclient_smsg_tail = NULL;
-
     /* response acking a request */
     p->tcph->th_ack = htonl(175);
     p->tcph->th_seq = htonl(328);
@@ -2837,6 +2382,7 @@ static int StreamTcpReassembleTest39 (void)
     p->payload = NULL;
     if (StreamTcpPacket(&tv, p, &stt, &pq) == -1)
         goto end;
+#if 0
     if (//!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server) ||
         //!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client) ||
         //f.alproto != ALPROTO_HTTP ||
@@ -2853,7 +2399,7 @@ static int StreamTcpReassembleTest39 (void)
         abort();
         goto end;
     }
-
+#endif
 
     ret = 1;
 end:
@@ -2888,7 +2434,6 @@ static int StreamTcpReassembleTest40 (void)
 
     StreamTcpInitConfig(TRUE);
     StreamTcpUTSetupSession(&ssn);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 130);
 
     TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(NULL);
 
@@ -2935,14 +2480,6 @@ static int StreamTcpReassembleTest40 (void)
         printf("failed in segments reassembly, while processing toserver packet (1): ");
         goto end;
     }
-
-    /* Check if we have stream smsgs in queue */
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOCLIENT) > 0) {
-        printf("there shouldn't be any stream smsgs in the queue, as we didn't"
-                " processed any smsg from toserver side till yet (2): ");
-        goto end;
-    }
-
     p->flowflags = FLOW_PKT_TOCLIENT;
     p->payload = httpbuf2;
     p->payload_len = httplen2;
@@ -2985,7 +2522,6 @@ static int StreamTcpReassembleTest40 (void)
     /* check is have the segment in the list and flagged or not */
     if (ssn.client.seg_list == NULL ||
         SEGMENT_BEFORE_OFFSET(&ssn.client, ssn.client.seg_list, STREAM_APP_PROGRESS(&ssn.client)))
-//        (ssn.client.seg_list->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED))
     {
         printf("the list is NULL or the processed segment has not been flaged (7): ");
         goto end;
@@ -3042,15 +2578,6 @@ static int StreamTcpReassembleTest40 (void)
         printf("failed in segments reassembly, while processing toserver packet (15): ");
         goto end;
     }
-
-    /* Check if we have stream smsgs in queue */
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) == 0) {
-        printf("there should be a stream smsgs in the queue, as we have detected"
-                " the app layer protocol and one smsg from toserver side has "
-                "been sent (16): ");
-        goto end;
-    }
-
     if (f->alproto != ALPROTO_HTTP) {
         printf("app layer proto has not been detected (18): ");
         goto end;
@@ -3067,219 +2594,20 @@ end:
     return ret;
 }
 
-/**
- *  \retval On success it returns 1 and on failure 0.
- */
-
-static int StreamTcpReassembleTest43 (void)
-{
-    int ret = 0;
-    Packet *p = PacketGetFromAlloc();
-    if (unlikely(p == NULL))
-        return 0;
-    Flow *f = NULL;
-    TCPHdr tcph;
-    TcpSession ssn;
-    PacketQueue pq;
-    memset(&pq,0,sizeof(PacketQueue));
-    memset(&tcph, 0, sizeof (TCPHdr));
-    ThreadVars tv;
-    memset(&tv, 0, sizeof (ThreadVars));
-
-    StreamTcpInitConfig(TRUE);
-    StreamTcpUTSetupSession(&ssn);
-    TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(NULL);
-
-    uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0";
-
-    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
-
-    uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
-    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
-
-    uint8_t httpbuf3[] = "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu"
-                         "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN"
-                         "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N"
-                         "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk"
-                         "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l"
-                         "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN"
-                         "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt"
-                         "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz"
-                         "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw"
-                         "aG9uZT\r\n\r\n";
-    uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
-
-    ssn.server.base_seq= 10;
-    ssn.server.isn = 9;
-    ssn.server.last_ack = 600;
-    ssn.client.base_seq = 10;
-    ssn.client.isn = 9;
-    ssn.client.last_ack = 600;
-
-    f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
-    if (f == NULL)
-        goto end;
-    f->protoctx = &ssn;
-    f->proto = IPPROTO_TCP;
-    p->flow = f;
-
-    tcph.th_win = htons(5480);
-    tcph.th_seq = htonl(10);
-    tcph.th_ack = htonl(10);
-    tcph.th_flags = TH_ACK|TH_PUSH;
-    p->tcph = &tcph;
-    p->flowflags = FLOW_PKT_TOCLIENT;
-
-    p->payload = httpbuf2;
-    p->payload_len = httplen2;
-    ssn.state = TCP_ESTABLISHED;
-
-    TcpStream *s = NULL;
-    s = &ssn.server;
-
-    FLOWLOCK_WRLOCK(f);
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
-        printf("failed in segments reassembly, while processing toserver packet (1): ");
-        goto end;
-    }
-
-    /* Check if we have stream smsgs in queue */
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) > 0) {
-        printf("there shouldn't be any stream smsgs in the queue (2): ");
-        goto end;
-    }
-
-    p->flowflags = FLOW_PKT_TOSERVER;
-    p->payload = httpbuf1;
-    p->payload_len = httplen1;
-    tcph.th_seq = htonl(10);
-    tcph.th_ack = htonl(55);
-    s = &ssn.client;
-
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
-        printf("failed in segments reassembly, while processing toserver packet (3): ");
-        goto end;
-    }
-
-    /* Check if we have stream smsgs in queue */
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOCLIENT) > 0) {
-        printf("there shouldn't be any stream smsgs in the queue, as we didn't"
-                " processed any smsg from toserver side till yet (4): ");
-        goto end;
-    }
-
-    p->flowflags = FLOW_PKT_TOCLIENT;
-    p->payload = httpbuf2;
-    p->payload_len = httplen2;
-    tcph.th_seq = htonl(55);
-    tcph.th_ack = htonl(44);
-    s = &ssn.server;
-
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
-        printf("failed in segments reassembly, while processing toserver packet (5): ");
-        goto end;
-    }
-    if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn.client)) {
-        printf("app layer detected flag isn't set, it should be (8): ");
-        goto end;
-    }
-
-    /* This packets induces a packet gap and also shows why we need to
-       process the current segment completely, even if it results in sending more
-       than one smsg to the app layer. If we don't send more than one smsg in
-       this case, then the first segment of lentgh 34 bytes will be sent to
-       app layer and protocol can not be detected in that message and moreover
-       the segment lentgh is less than the max. signature size for protocol
-       detection, so this will keep looping !! */
-    p->flowflags = FLOW_PKT_TOSERVER;
-    p->payload = httpbuf3;
-    p->payload_len = httplen3;
-    tcph.th_seq = htonl(54);
-    tcph.th_ack = htonl(100);
-    s = &ssn.client;
-
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
-        printf("failed in segments reassembly, while processing toserver packet (9): ");
-        goto end;
-    }
-
-    /* Check if we have stream smsgs in queue */
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOCLIENT) > 0) {
-        printf("there shouldn't be any stream smsgs in the queue, as we didn't"
-                " detected the app layer protocol till yet (10): ");
-        goto end;
-    }
-
-    p->flowflags = FLOW_PKT_TOCLIENT;
-    p->payload = httpbuf2;
-    p->payload_len = httplen2;
-    tcph.th_seq = htonl(100);
-    tcph.th_ack = htonl(53);
-    s = &ssn.server;
-
-    if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) {
-        printf("failed in segments reassembly, while processing toserver packet (11): ");
-        goto end;
-    }
-    /* the flag should be set, as the smsg scanned size has crossed the max.
-       signature size for app proto detection */
-    if (!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn.client)) {
-        printf("app layer detected flag is not set, it should be (14): ");
-        goto end;
-    }
-
-    ret = 1;
-end:
-    StreamTcpUTClearSession(&ssn);
-    StreamTcpReassembleFreeThreadCtx(ra_ctx);
-    StreamTcpFreeConfig(TRUE);
-    SCFree(p);
-    SCMutexUnlock(&f->m);
-    UTHFreeFlow(f);
-    return ret;
-}
-
 /** \test   Test the memcap incrementing/decrementing and memcap check */
 static int StreamTcpReassembleTest44(void)
 {
-    uint8_t ret = 0;
     StreamTcpInitConfig(TRUE);
     uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
-
     StreamTcpReassembleIncrMemuse(500);
-    if (SC_ATOMIC_GET(ra_memuse) != (memuse+500)) {
-        printf("failed in incrementing the memory");
-        goto end;
-    }
-
+    FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
     StreamTcpReassembleDecrMemuse(500);
-    if (SC_ATOMIC_GET(ra_memuse) != memuse) {
-        printf("failed in decrementing the memory");
-        goto end;
-    }
-
-    if (StreamTcpReassembleCheckMemcap(500) != 1) {
-        printf("failed in validating the memcap");
-        goto end;
-    }
-
-    if (StreamTcpReassembleCheckMemcap((memuse + stream_config.reassembly_memcap)) != 0) {
-        printf("failed in validating the memcap");
-        goto end;
-    }
-
-    StreamTcpFreeConfig(TRUE);
-
-    if (SC_ATOMIC_GET(ra_memuse) != 0) {
-        printf("failed in clearing the memory");
-        goto end;
-    }
-
-    ret = 1;
-    return ret;
-end:
+    FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
+    FAIL_IF(StreamTcpReassembleCheckMemcap(500) != 1);
+    FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + stream_config.reassembly_memcap)) != 0);
     StreamTcpFreeConfig(TRUE);
-    return ret;
+    FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
+    PASS;
 }
 
 /**
@@ -3387,11 +2715,6 @@ static int StreamTcpReassembleTest47 (void)
     memset(&pq,0,sizeof(PacketQueue));
     memset(&tcph, 0, sizeof (TCPHdr));
     memset(&tv, 0, sizeof (ThreadVars));
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0);
-
     StreamTcpInitConfig(TRUE);
     StreamTcpUTSetupSession(&ssn);
     TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(NULL);
@@ -3467,7 +2790,6 @@ static int StreamTcpReassembleInlineTest01(void)
     StreamTcpUTSetupStream(&ssn.client, 1);
     FLOW_INITIALIZE(&f);
 
-    uint8_t stream_payload[] = "AAAAABBBBBCCCCC";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (p == NULL) {
@@ -3491,19 +2813,6 @@ static int StreamTcpReassembleInlineTest01(void)
         goto end;
     }
     ssn.client.next_seq = 17;
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    FAIL_IF(UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1);
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload, 15) == 0)
-        goto end;
-
     ret = 1;
 end:
     FLOWLOCK_UNLOCK(&f);
@@ -3533,8 +2842,6 @@ static int StreamTcpReassembleInlineTest02(void)
     StreamTcpUTSetupStream(&ssn.client, 1);
     FLOW_INITIALIZE(&f);
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCC";
-    uint8_t stream_payload2[] = "AAAAABBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (p == NULL) {
@@ -3558,43 +2865,11 @@ static int StreamTcpReassembleInlineTest02(void)
         goto end;
     }
     ssn.client.next_seq = 17;
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1) {
-        printf("expected a single stream message: ");
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 15) == 0)
-        goto end;
-
     if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
         printf("failed to add segment 4: ");
         goto end;
     }
     ssn.client.next_seq = 22;
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed 2: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected a single stream message: ");
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 20) == 0)
-        goto end;
-
     ret = 1;
 end:
     FLOWLOCK_UNLOCK(&f);
@@ -3627,8 +2902,6 @@ static int StreamTcpReassembleInlineTest03(void)
 
     stream_config.reassembly_toserver_chunk_size = 15;
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCC";
-    uint8_t stream_payload2[] = "BBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (p == NULL) {
@@ -3653,22 +2926,6 @@ static int StreamTcpReassembleInlineTest03(void)
         goto end;
     }
     ssn.client.next_seq = 17;
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1) {
-        printf("expected a single stream message 1, got %u: ", UtSsnSmsgCnt(&ssn, STREAM_TOSERVER));
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 15) == 0)
-        goto end;
-
     if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
         printf("failed to add segment 4: ");
         goto end;
@@ -3676,22 +2933,6 @@ static int StreamTcpReassembleInlineTest03(void)
     ssn.client.next_seq = 22;
 
     p->tcph->th_seq = htonl(17);
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed 2: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected two stream messages: ");
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 15) == 0)
-        goto end;
-
     ret = 1;
 end:
     FLOWLOCK_UNLOCK(&f);
@@ -3724,8 +2965,6 @@ static int StreamTcpReassembleInlineTest04(void)
 
     stream_config.reassembly_toserver_chunk_size = 16;
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCC";
-    uint8_t stream_payload2[] = "ABBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (p == NULL) {
@@ -3750,22 +2989,6 @@ static int StreamTcpReassembleInlineTest04(void)
         goto end;
     }
     ssn.client.next_seq = 17;
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1) {
-        printf("expected a single stream message: ");
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 15) == 0)
-        goto end;
-
     if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
         printf("failed to add segment 4: ");
         goto end;
@@ -3773,305 +2996,6 @@ static int StreamTcpReassembleInlineTest04(void)
     ssn.client.next_seq = 22;
 
     p->tcph->th_seq = htonl(17);
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed 2: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected a single stream message: ");
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 16) == 0)
-        goto end;
-
-    ret = 1;
-end:
-    FLOWLOCK_UNLOCK(&f);
-    FLOW_DESTROY(&f);
-    UTHFreePacket(p);
-    StreamTcpUTClearSession(&ssn);
-    StreamTcpUTDeinit(ra_ctx);
-    return ret;
-}
-
-/** \test with a GAP we should have 2 smsgs */
-static int StreamTcpReassembleInlineTest05(void)
-{
-    int ret = 0;
-    TcpReassemblyThreadCtx *ra_ctx = NULL;
-    ThreadVars tv;
-    TcpSession ssn;
-    Flow f;
-
-    memset(&tv, 0x00, sizeof(tv));
-
-    StreamTcpUTInit(&ra_ctx);
-    StreamTcpUTInitInline();
-    StreamTcpUTSetupSession(&ssn);
-    StreamTcpUTSetupStream(&ssn.client, 1);
-    FLOW_INITIALIZE(&f);
-
-    uint8_t stream_payload1[] = "AAAAABBBBB";
-    uint8_t stream_payload2[] = "DDDDD";
-    uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
-    Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
-    if (p == NULL) {
-        printf("couldn't get a packet: ");
-        goto end;
-    }
-    p->tcph->th_seq = htonl(12);
-    p->flow = &f;
-
-    FLOWLOCK_WRLOCK(&f);
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  2, 'A', 5) == -1) {
-        printf("failed to add segment 1: ");
-        goto end;
-    }
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  7, 'B', 5) == -1) {
-        printf("failed to add segment 2: ");
-        goto end;
-    }
-    ssn.client.next_seq = 12;
-
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
-        printf("failed to add segment 4: ");
-        goto end;
-    }
-
-    p->tcph->th_seq = htonl(17);
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected a single stream message: ");
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 10) == 0)
-        goto end;
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 5) == 0)
-        goto end;
-
-    ret = 1;
-end:
-    FLOWLOCK_UNLOCK(&f);
-    FLOW_DESTROY(&f);
-    UTHFreePacket(p);
-    StreamTcpUTClearSession(&ssn);
-    StreamTcpUTDeinit(ra_ctx);
-    return ret;
-}
-
-/** \test with a GAP we should have 2 smsgs, with filling the GAP later */
-static int StreamTcpReassembleInlineTest06(void)
-{
-    int ret = 0;
-    TcpReassemblyThreadCtx *ra_ctx = NULL;
-    ThreadVars tv;
-    TcpSession ssn;
-    Flow f;
-
-    memset(&tv, 0x00, sizeof(tv));
-
-    StreamTcpUTInit(&ra_ctx);
-    StreamTcpUTInitInline();
-    StreamTcpUTSetupSession(&ssn);
-    StreamTcpUTSetupStream(&ssn.client, 1);
-    FLOW_INITIALIZE(&f);
-
-    uint8_t stream_payload1[] = "AAAAABBBBB";
-    uint8_t stream_payload2[] = "DDDDD";
-    uint8_t stream_payload3[] = "AAAAABBBBBCCCCCDDDDD";
-    uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
-    Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
-    if (p == NULL) {
-        printf("couldn't get a packet: ");
-        goto end;
-    }
-    p->tcph->th_seq = htonl(12);
-    p->flow = &f;
-
-    FLOWLOCK_WRLOCK(&f);
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  2, 'A', 5) == -1) {
-        printf("failed to add segment 1: ");
-        goto end;
-    }
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  7, 'B', 5) == -1) {
-        printf("failed to add segment 2: ");
-        goto end;
-    }
-    ssn.client.next_seq = 12;
-
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
-        printf("failed to add segment 4: ");
-        goto end;
-    }
-
-    p->tcph->th_seq = htonl(17);
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected two stream messages: ");
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 10) == 0)
-        goto end;
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 5) == 0)
-        goto end;
-
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
-        printf("failed to add segment 3: ");
-        goto end;
-    }
-    ssn.client.next_seq = 22;
-
-    p->tcph->th_seq = htonl(12);
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 3) {
-        printf("expected a single stream message, got %u: ", UtSsnSmsgCnt(&ssn, STREAM_TOSERVER));
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next->next;
-    if (UtTestSmsg(smsg, stream_payload3, 20) == 0)
-        goto end;
-
-    ret = 1;
-end:
-    FLOWLOCK_UNLOCK(&f);
-    FLOW_DESTROY(&f);
-    UTHFreePacket(p);
-    StreamTcpUTClearSession(&ssn);
-    StreamTcpUTDeinit(ra_ctx);
-    return ret;
-}
-
-/** \test with a GAP we should have 2 smsgs, with filling the GAP later, small
- *        window */
-static int StreamTcpReassembleInlineTest07(void)
-{
-    int ret = 0;
-    TcpReassemblyThreadCtx *ra_ctx = NULL;
-    ThreadVars tv;
-    TcpSession ssn;
-    Flow f;
-
-    memset(&tv, 0x00, sizeof(tv));
-
-    StreamTcpUTInit(&ra_ctx);
-    StreamTcpUTInitInline();
-    StreamTcpUTSetupSession(&ssn);
-    StreamTcpUTSetupStream(&ssn.client, 1);
-    FLOW_INITIALIZE(&f);
-
-    stream_config.reassembly_toserver_chunk_size = 16;
-
-    uint8_t stream_payload1[] = "ABBBBB";
-    uint8_t stream_payload2[] = "DDDDD";
-    uint8_t stream_payload3[] = "AAAAABBBBBCCCCCD";
-    uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
-    Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
-    if (p == NULL) {
-        printf("couldn't get a packet: ");
-        goto end;
-    }
-    p->tcph->th_seq = htonl(12);
-    p->flow = &f;
-    p->flowflags |= FLOW_PKT_TOSERVER;
-
-    FLOWLOCK_WRLOCK(&f);
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  2, 'A', 5) == -1) {
-        printf("failed to add segment 1: ");
-        goto end;
-    }
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  7, 'B', 5) == -1) {
-        printf("failed to add segment 2: ");
-        goto end;
-    }
-    ssn.client.next_seq = 12;
-
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
-        printf("failed to add segment 4: ");
-        goto end;
-    }
-
-    p->tcph->th_seq = htonl(17);
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected a single stream message, got %u: ", UtSsnSmsgCnt(&ssn, STREAM_TOSERVER));
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 6) == 0) {
-        printf("stream_payload1 failed: ");
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 5) == 0) {
-        printf("stream_payload2 failed: ");
-        goto end;
-    }
-
-    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
-        printf("failed to add segment 3: ");
-        goto end;
-    }
-    ssn.client.next_seq = 22;
-
-    p->tcph->th_seq = htonl(12);
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 3) {
-        printf("expected a single stream message, got %u: ", UtSsnSmsgCnt(&ssn, STREAM_TOSERVER));
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next->next;
-    if (UtTestSmsg(smsg, stream_payload3, 16) == 0) {
-        printf("stream_payload3 failed: ");
-        goto end;
-    }
-
     ret = 1;
 end:
     FLOWLOCK_UNLOCK(&f);
@@ -4104,8 +3028,6 @@ static int StreamTcpReassembleInlineTest08(void)
     ssn.client.flags |= STREAMTCP_STREAM_FLAG_GAP;
     f.protoctx = &ssn;
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCC";
-    uint8_t stream_payload2[] = "BBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     FAIL_IF(p == NULL);
@@ -4117,33 +3039,9 @@ static int StreamTcpReassembleInlineTest08(void)
     FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client,  7, 'B', 5) == -1);
     FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
     ssn.client.next_seq = 17;
-
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    FAIL_IF(r < 0);
-
-    FAIL_IF(UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1);
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    FAIL_IF(UtTestSmsg(smsg, stream_payload1, 15) == 0);
-
-    FAIL_IF(STREAM_RAW_PROGRESS(&ssn.client) != 15);
     FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
     ssn.client.next_seq = 22;
     p->tcph->th_seq = htonl(17);
-
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    FAIL_IF (r < 0);
-
-    FAIL_IF(UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2);
-
-    smsg = ssn.toserver_smsg_head->next;
-    FAIL_IF(UtTestSmsg(smsg, stream_payload2, 15) == 0);
-
-    FAIL_IF(STREAM_RAW_PROGRESS(&ssn.client) != 20);
-
-    smsg = ssn.toserver_smsg_head;
-    StreamMsgReturnToPool(smsg);
-    ssn.toserver_smsg_head = ssn.toserver_smsg_head->next;
-
     StreamTcpPruneSession(&f, STREAM_TOSERVER);
 
     FAIL_IF (ssn.client.seg_list->seq != 2);
@@ -4179,9 +3077,6 @@ static int StreamTcpReassembleInlineTest09(void)
     stream_config.reassembly_toserver_chunk_size = 20;
     ssn.client.flags |= STREAMTCP_STREAM_FLAG_GAP;
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCC";
-    uint8_t stream_payload2[] = "DDDDD";
-    uint8_t stream_payload3[] = "AAAAABBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     if (p == NULL) {
@@ -4208,27 +3103,6 @@ static int StreamTcpReassembleInlineTest09(void)
     ssn.client.next_seq = 12;
     ssn.client.last_ack = 10;
 
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 2) {
-        printf("expected 2 stream message2, got %u: ", UtSsnSmsgCnt(&ssn, STREAM_TOSERVER));
-        goto end;
-    }
-
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    if (UtTestSmsg(smsg, stream_payload1, 10) == 0)
-        goto end;
-
-    smsg = ssn.toserver_smsg_head->next;
-    if (UtTestSmsg(smsg, stream_payload2, 5) == 0)
-        goto end;
-
-    FAIL_IF(STREAM_RAW_PROGRESS(&ssn.client) != 10);
-
     /* close the GAP and see if we properly reassemble and update base_seq */
     if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
         printf("failed to add segment 4: ");
@@ -4238,23 +3112,6 @@ static int StreamTcpReassembleInlineTest09(void)
 
     p->tcph->th_seq = htonl(12);
 
-    r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    if (r < 0) {
-        printf("StreamTcpReassembleInlineRaw failed 2: ");
-        goto end;
-    }
-
-    if (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 3) {
-        printf("expected 3 stream messages: ");
-        goto end;
-    }
-
-    smsg = ssn.toserver_smsg_head->next->next;
-    if (UtTestSmsg(smsg, stream_payload3, 20) == 0)
-        goto end;
-
-    FAIL_IF(STREAM_RAW_PROGRESS(&ssn.client) != 20);
-
     if (ssn.client.seg_list->seq != 2) {
         printf("expected segment 1 (seq 2) to be first in the list, got seq %"PRIu32": ", ssn.client.seg_list->seq);
         goto end;
@@ -4378,7 +3235,6 @@ static int StreamTcpReassembleInsertTest01(void)
     ssn.client.os_policy = OS_POLICY_LAST;
     FLOW_INITIALIZE(&f);
 
-    uint8_t stream_payload1[] = "AAAAABBBBBCCCCCDDDDD";
     uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
     Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
     FAIL_IF(p == NULL);
@@ -4392,13 +3248,6 @@ static int StreamTcpReassembleInsertTest01(void)
     FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
     ssn.client.next_seq = 21;
 
-    int r = StreamTcpReassembleRaw(&tv, ra_ctx, &ssn, &ssn.client, p);
-    FAIL_IF (r < 0);
-    FAIL_IF (UtSsnSmsgCnt(&ssn, STREAM_TOSERVER) != 1);
-    StreamMsg *smsg = ssn.toserver_smsg_head;
-    FAIL_IF(UtTestSmsg(smsg, stream_payload1, 20) == 0);
-    FAIL_IF(STREAM_RAW_PROGRESS(&ssn.client) != 20);
-
     FLOW_DESTROY(&f);
     UTHFreePacket(p);
     StreamTcpUTClearSession(&ssn);
@@ -4517,14 +3366,10 @@ void StreamTcpReassembleRegisterTests(void)
                    StreamTcpReassembleTest34);
     UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
                    StreamTcpReassembleTest37);
-    UtRegisterTest("StreamTcpReassembleTest38 -- app proto test",
-                   StreamTcpReassembleTest38);
     UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
                    StreamTcpReassembleTest39);
     UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
                    StreamTcpReassembleTest40);
-    UtRegisterTest("StreamTcpReassembleTest43 -- min smsg size test",
-                   StreamTcpReassembleTest43);
     UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
                    StreamTcpReassembleTest44);
     UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
@@ -4542,12 +3387,6 @@ void StreamTcpReassembleRegisterTests(void)
                    StreamTcpReassembleInlineTest03);
     UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
                    StreamTcpReassembleInlineTest04);
-    UtRegisterTest("StreamTcpReassembleInlineTest05 -- inline RAW ra 5 GAP",
-                   StreamTcpReassembleInlineTest05);
-    UtRegisterTest("StreamTcpReassembleInlineTest06 -- inline RAW ra 6 GAP",
-                   StreamTcpReassembleInlineTest06);
-    UtRegisterTest("StreamTcpReassembleInlineTest07 -- inline RAW ra 7 GAP",
-                   StreamTcpReassembleInlineTest07);
     UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
                    StreamTcpReassembleInlineTest08);
     UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h
index 8f3a3a6803..f2bbf277e5 100644
--- a/src/stream-tcp-reassemble.h
+++ b/src/stream-tcp-reassemble.h
@@ -118,5 +118,7 @@ int StreamTcpAppLayerIsDisabled(Flow *f);
 int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
 #endif
 
+bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p);
+
 #endif /* __STREAM_TCP_REASSEMBLE_H__ */
 
diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index 126c7fde72..82cfcfd10d 100644
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -187,8 +187,6 @@ void StreamTcpStreamCleanup(TcpStream *stream)
 void StreamTcpSessionCleanup(TcpSession *ssn)
 {
     SCEnter();
-
-    StreamMsg *smsg = NULL;
     TcpStateQueue *q, *q_next;
 
     if (ssn == NULL)
@@ -197,29 +195,6 @@ void StreamTcpSessionCleanup(TcpSession *ssn)
     StreamTcpStreamCleanup(&ssn->client);
     StreamTcpStreamCleanup(&ssn->server);
 
-    /* if we have (a) smsg(s), return to the pool */
-    smsg = ssn->toserver_smsg_head;
-    while(smsg != NULL) {
-        StreamMsg *smsg_next = smsg->next;
-        SCLogDebug("returning smsg %p to pool", smsg);
-        smsg->next = NULL;
-        smsg->prev = NULL;
-        StreamMsgReturnToPool(smsg);
-        smsg = smsg_next;
-    }
-    ssn->toserver_smsg_head = NULL;
-
-    smsg = ssn->toclient_smsg_head;
-    while(smsg != NULL) {
-        StreamMsg *smsg_next = smsg->next;
-        SCLogDebug("returning smsg %p to pool", smsg);
-        smsg->next = NULL;
-        smsg->prev = NULL;
-        StreamMsgReturnToPool(smsg);
-        smsg = smsg_next;
-    }
-    ssn->toclient_smsg_head = NULL;
-
     q = ssn->queue;
     while (q != NULL) {
         q_next = q->next;
@@ -553,9 +528,6 @@ void StreamTcpInitConfig(char quiet)
             (int) (stream_config.reassembly_toserver_chunk_size *
                    (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
     }
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER,
-            stream_config.reassembly_toserver_chunk_size);
-
     char *temp_stream_reassembly_toclient_chunk_size_str;
     if (ConfGet("stream.reassembly.toclient-chunk-size",
                 &temp_stream_reassembly_toclient_chunk_size_str) == 1) {
@@ -578,10 +550,6 @@ void StreamTcpInitConfig(char quiet)
             (int) (stream_config.reassembly_toclient_chunk_size *
                    (r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
     }
-
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT,
-            stream_config.reassembly_toclient_chunk_size);
-
     if (!quiet) {
         SCLogConfig("stream.reassembly \"toserver-chunk-size\": %"PRIu16,
             stream_config.reassembly_toserver_chunk_size);
@@ -593,8 +561,6 @@ void StreamTcpInitConfig(char quiet)
     if (ConfGetBool("stream.reassembly.raw", &enable_raw) == 1) {
         if (!enable_raw) {
             stream_config.ssn_init_flags = STREAMTCP_FLAG_DISABLE_RAW;
-// TODO how to handle this now?
-//            stream_config.segment_init_flags = SEGMENTTCP_FLAG_RAW_PROCESSED;
         }
     } else {
         enable_raw = 1;
@@ -6170,11 +6136,6 @@ static int StreamTcpTest05 (void)
     int ret = 0;
 
     StreamTcpUTInit(&stt.ra_ctx);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     tcph.th_win = htons(5480);
     tcph.th_seq = htonl(10);
     tcph.th_ack = htonl(20);
@@ -7004,11 +6965,6 @@ static int StreamTcpTest14 (void)
     strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
     ip_addr = StreamTcpParseOSPolicy(os_policy_name);
     SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     addr.s_addr = inet_addr("192.168.0.1");
     tcph.th_win = htons(5480);
     tcph.th_seq = htonl(10);
@@ -7411,11 +7367,6 @@ static int StreamTcpTest15 (void)
     strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
     ip_addr = StreamTcpParseOSPolicy(os_policy_name);
     SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     addr.s_addr = inet_addr("192.168.0.20");
     tcph.th_win = htons(5480);
     tcph.th_seq = htonl(10);
@@ -7583,11 +7534,6 @@ static int StreamTcpTest16 (void)
     strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
     ip_addr = StreamTcpParseOSPolicy(os_policy_name);
     SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     addr.s_addr = inet_addr("192.168.0.1");
     tcph.th_win = htons(5480);
     tcph.th_seq = htonl(10);
@@ -7758,11 +7704,6 @@ static int StreamTcpTest17 (void)
     strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
     ip_addr = StreamTcpParseOSPolicy(os_policy_name);
     SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     addr.s_addr = inet_addr("192.168.0.1");
     tcph.th_win = htons(5480);
     tcph.th_seq = htonl(10);
@@ -8164,11 +8105,6 @@ static int StreamTcpTest23(void)
 
     StreamTcpUTInit(&stt.ra_ctx);
     StreamTcpUTSetupSession(&ssn);
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     FLOW_INITIALIZE(&f);
     ssn.client.os_policy = OS_POLICY_BSD;
     f.protoctx = &ssn;
@@ -8308,11 +8244,6 @@ static int StreamTcpTest25(void)
     tcph.th_flags = TH_SYN | TH_CWR;
     p->tcph = &tcph;
     p->flowflags = FLOW_PKT_TOSERVER;
-
-    /* prevent L7 from kicking in */
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
-    StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
-
     StreamTcpUTInit(&stt.ra_ctx);
 
     if (StreamTcpPacket(&tv, p, &stt, &pq) == -1)
@@ -9132,10 +9063,8 @@ static int StreamTcpTest33 (void)
     StreamTcpThread stt;
     TCPHdr tcph;
     TcpReassemblyThreadCtx ra_ctx;
-    StreamMsgQueue stream_q;
     PacketQueue pq;
     memset(&pq,0,sizeof(PacketQueue));
-    memset(&stream_q, 0, sizeof(StreamMsgQueue));
     memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
     memset (&p, 0, SIZE_OF_PACKET);
     memset (&f, 0, sizeof(Flow));
@@ -9150,7 +9079,6 @@ static int StreamTcpTest33 (void)
     p.tcph = &tcph;
     p.flowflags = FLOW_PKT_TOSERVER;
     int ret = 0;
-    ra_ctx.stream_q = &stream_q;
     stt.ra_ctx = &ra_ctx;
 
     StreamTcpInitConfig(TRUE);
@@ -9236,10 +9164,8 @@ static int StreamTcpTest34 (void)
     StreamTcpThread stt;
     TCPHdr tcph;
     TcpReassemblyThreadCtx ra_ctx;
-    StreamMsgQueue stream_q;
     PacketQueue pq;
     memset(&pq,0,sizeof(PacketQueue));
-    memset(&stream_q, 0, sizeof(StreamMsgQueue));
     memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
     memset (&p, 0, SIZE_OF_PACKET);
     memset (&f, 0, sizeof(Flow));
@@ -9254,7 +9180,6 @@ static int StreamTcpTest34 (void)
     p.tcph = &tcph;
     p.flowflags = FLOW_PKT_TOSERVER;
     int ret = 0;
-    ra_ctx.stream_q = &stream_q;
     stt.ra_ctx = &ra_ctx;
 
     StreamTcpInitConfig(TRUE);
@@ -9304,10 +9229,8 @@ static int StreamTcpTest35 (void)
     StreamTcpThread stt;
     TCPHdr tcph;
     TcpReassemblyThreadCtx ra_ctx;
-    StreamMsgQueue stream_q;
     PacketQueue pq;
     memset(&pq,0,sizeof(PacketQueue));
-    memset(&stream_q, 0, sizeof(StreamMsgQueue));
     memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
     memset (&p, 0, SIZE_OF_PACKET);
     memset (&f, 0, sizeof(Flow));
@@ -9322,7 +9245,6 @@ static int StreamTcpTest35 (void)
     p.tcph = &tcph;
     p.flowflags = FLOW_PKT_TOSERVER;
     int ret = 0;
-    ra_ctx.stream_q = &stream_q;
     stt.ra_ctx = &ra_ctx;
 
     StreamTcpInitConfig(TRUE);
@@ -9554,7 +9476,7 @@ static int StreamTcpTest37(void)
     }
 
     TcpStream *stream = &(((TcpSession *)p->flow->protoctx)->client);
-    FAIL_IF(STREAM_RAW_PROGRESS(stream) != 3);
+    FAIL_IF(STREAM_RAW_PROGRESS(stream) != 0); // no detect no progress update
 
     StreamTcpSessionClear(p->flow->protoctx);
 
diff --git a/src/stream-tcp.h b/src/stream-tcp.h
index 98afac2737..a04857515e 100644
--- a/src/stream-tcp.h
+++ b/src/stream-tcp.h
@@ -45,7 +45,6 @@ typedef struct TcpStreamCnf_ {
     uint64_t reassembly_memcap; /**< max memory usage for stream reassembly */
 
     uint32_t ssn_init_flags; /**< new ssn flags will be initialized to this */
-    uint8_t segment_init_flags; /**< new seg flags will be initialized to this */
 
     uint32_t prealloc_sessions; /**< ssns to prealloc per stream thread */
     uint32_t prealloc_segments; /**< segments to prealloc per stream thread */
@@ -118,6 +117,12 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag,
 void StreamTcpReassembleConfigEnableOverlapCheck(void);
 void TcpSessionSetReassemblyDepth(TcpSession *ssn, uint32_t size);
 
+typedef int (*StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len);
+
+int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
+        StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out);
+void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress);
+
 /** ------- Inline functions: ------ */
 
 /**
diff --git a/src/stream.c b/src/stream.c
index 6415420981..a09f125e35 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2017 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -19,8 +19,6 @@
  * \file
  *
  * \author Victor Julien <victor@inliniac.net>
- *
- * Stream Chunk Handling API
  */
 
 #include "suricata-common.h"
@@ -32,241 +30,6 @@
 #include "stream-tcp.h"
 #include "flow-util.h"
 
-#ifdef DEBUG
-static SCMutex stream_pool_memuse_mutex;
-static uint64_t stream_pool_memuse = 0;
-static uint64_t stream_pool_memcnt = 0;
-#endif
-
-/* per queue setting */
-static uint16_t toserver_min_chunk_len = 2560;
-static uint16_t toclient_min_chunk_len = 2560;
-
-static Pool *stream_msg_pool = NULL;
-static SCMutex stream_msg_pool_mutex = SCMUTEX_INITIALIZER;
-
-static void StreamMsgEnqueue (StreamMsgQueue *q, StreamMsg *s)
-{
-    SCEnter();
-    SCLogDebug("s %p", s);
-    /* more packets in queue */
-    if (q->top != NULL) {
-        s->next = q->top;
-        q->top->prev = s;
-        q->top = s;
-    /* only packet */
-    } else {
-        q->top = s;
-        q->bot = s;
-    }
-    q->len++;
-#ifdef DBG_PERF
-    if (q->len > q->dbg_maxlen)
-        q->dbg_maxlen = q->len;
-#endif /* DBG_PERF */
-    SCReturn;
-}
-
-static StreamMsg *StreamMsgDequeue (StreamMsgQueue *q)
-{
-    SCEnter();
-
-    /* if the queue is empty there are no packets left.
-     * In that case we sleep and try again. */
-    if (q->len == 0) {
-        SCReturnPtr(NULL, "StreamMsg");
-    }
-
-    /* pull the bottom packet from the queue */
-    StreamMsg *s = q->bot;
-
-    /* more packets in queue */
-    if (q->bot->prev != NULL) {
-        q->bot = q->bot->prev;
-        q->bot->next = NULL;
-        /* just the one we remove, so now empty */
-    } else {
-        q->top = NULL;
-        q->bot = NULL;
-    }
-    q->len--;
-
-    s->next = NULL;
-    s->prev = NULL;
-    SCReturnPtr(s, "StreamMsg");
-}
-
-/* Used by stream reassembler to get msgs */
-StreamMsg *StreamMsgGetFromPool(void)
-{
-    SCMutexLock(&stream_msg_pool_mutex);
-    StreamMsg *s = (StreamMsg *)PoolGet(stream_msg_pool);
-    SCMutexUnlock(&stream_msg_pool_mutex);
-    return s;
-}
-
-/* Used by l7inspection to return msgs to pool */
-void StreamMsgReturnToPool(StreamMsg *s)
-{
-    SCLogDebug("s %p", s);
-    SCMutexLock(&stream_msg_pool_mutex);
-    PoolReturn(stream_msg_pool, (void *)s);
-    SCMutexUnlock(&stream_msg_pool_mutex);
-}
-
-/* Used by l7inspection to get msgs with data */
-StreamMsg *StreamMsgGetFromQueue(StreamMsgQueue *q)
-{
-    if (q->len > 0) {
-        StreamMsg *s = StreamMsgDequeue(q);
-        return s;
-    } else {
-        /* return NULL if we have no stream msg. Should only happen on signals. */
-        return NULL;
-    }
-}
-
-/* Used by stream reassembler to fill the queue for l7inspect reading */
-void StreamMsgPutInQueue(StreamMsgQueue *q, StreamMsg *s)
-{
-    StreamMsgEnqueue(q, s);
-    SCLogDebug("q->len %" PRIu32 "", q->len);
-}
-
-#define SIZE 4072
-void *StreamMsgPoolAlloc(void)
-{
-    if (StreamTcpReassembleCheckMemcap((uint32_t)(sizeof(StreamMsg)+SIZE)) == 0)
-        return NULL;
-
-    StreamMsg *m = SCCalloc(1, (sizeof(StreamMsg) + SIZE));
-    if (m != NULL) {
-        m->data = (uint8_t *)m + sizeof(StreamMsg);
-        m->data_size = SIZE;
-
-        StreamTcpReassembleIncrMemuse((uint32_t)(sizeof(StreamMsg)+SIZE));
-    }
-
-    return m;
-}
-
-int StreamMsgInit(void *data, void *initdata)
-{
-    StreamMsg *s = data;
-    memset(s->data, 0, s->data_size);
-
-#ifdef DEBUG
-    SCMutexLock(&stream_pool_memuse_mutex);
-    stream_pool_memuse += (sizeof(StreamMsg) + SIZE);
-    stream_pool_memcnt ++;
-    SCMutexUnlock(&stream_pool_memuse_mutex);
-#endif
-    return 1;
-}
-
-void StreamMsgPoolFree(void *ptr)
-{
-    if (ptr) {
-        SCFree(ptr);
-        StreamTcpReassembleDecrMemuse((uint32_t)(sizeof(StreamMsg)+SIZE));
-    }
-}
-
-void StreamMsgQueuesInit(uint32_t prealloc)
-{
-#ifdef DEBUG
-    SCMutexInit(&stream_pool_memuse_mutex, NULL);
-#endif
-    SCMutexLock(&stream_msg_pool_mutex);
-    stream_msg_pool = PoolInit(0, prealloc, 0,
-            StreamMsgPoolAlloc,StreamMsgInit,
-            NULL,NULL,StreamMsgPoolFree);
-    if (stream_msg_pool == NULL)
-        exit(EXIT_FAILURE); /* XXX */
-    SCMutexUnlock(&stream_msg_pool_mutex);
-}
-
-void StreamMsgQueuesDeinit(char quiet)
-{
-    if (quiet == FALSE) {
-        if (stream_msg_pool->max_outstanding > stream_msg_pool->allocated)
-            SCLogInfo("TCP segment chunk pool had a peak use of %u chunks, "
-                    "more than the prealloc setting of %u",
-                    stream_msg_pool->max_outstanding, stream_msg_pool->allocated);
-    }
-
-    SCMutexLock(&stream_msg_pool_mutex);
-    PoolFree(stream_msg_pool);
-    SCMutexUnlock(&stream_msg_pool_mutex);
-
-#ifdef DEBUG
-    SCMutexDestroy(&stream_pool_memuse_mutex);
-
-    if (quiet == FALSE)
-        SCLogDebug("stream_pool_memuse %"PRIu64", stream_pool_memcnt %"PRIu64"", stream_pool_memuse, stream_pool_memcnt);
-#endif
-}
-
-/** \brief alloc a stream msg queue
- *  \retval smq ptr to the queue or NULL */
-StreamMsgQueue *StreamMsgQueueGetNew(void)
-{
-    if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(StreamMsgQueue)) == 0)
-        return NULL;
-
-    StreamMsgQueue *smq = SCMalloc(sizeof(StreamMsgQueue));
-    if (unlikely(smq == NULL))
-        return NULL;
-
-    StreamTcpReassembleIncrMemuse((uint32_t)sizeof(StreamMsgQueue));
-
-    memset(smq, 0x00, sizeof(StreamMsgQueue));
-    return smq;
-}
-
-/** \brief Free a StreamMsgQueue
- *  \param q the queue to free
- *  \todo we may want to consider non empty queue's
- */
-void StreamMsgQueueFree(StreamMsgQueue *q)
-{
-    SCFree(q);
-    StreamTcpReassembleDecrMemuse((uint32_t)sizeof(StreamMsgQueue));
-}
-
-void StreamMsgQueueSetMinChunkLen(uint8_t dir, uint16_t len)
-{
-    if (dir == FLOW_PKT_TOSERVER) {
-        toserver_min_chunk_len = len;
-    } else {
-        toclient_min_chunk_len = len;
-    }
-}
-
-uint16_t StreamMsgQueueGetMinChunkLen(uint8_t dir)
-{
-    if (dir == FLOW_PKT_TOSERVER) {
-        return toserver_min_chunk_len;
-    } else {
-        return toclient_min_chunk_len;
-    }
-}
-
-/** \brief Return a list of smsgs to the pool */
-void StreamMsgReturnListToPool(void *list)
-{
-    /* if we have (a) smsg(s), return to the pool */
-    StreamMsg *smsg = (StreamMsg *)list;
-    while (smsg != NULL) {
-        StreamMsg *smsg_next = smsg->next;
-        SCLogDebug("returning smsg %p to pool", smsg);
-        smsg->next = NULL;
-        smsg->prev = NULL;
-        StreamMsgReturnToPool(smsg);
-        smsg = smsg_next;
-    }
-}
-
 /** \brief Run callback for all segments
  *
  * Must be called under flow lock.
diff --git a/src/stream.h b/src/stream.h
index 72674f803d..d421c99a70 100644
--- a/src/stream.h
+++ b/src/stream.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2017 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -33,43 +33,6 @@
 #define STREAM_GAP              0x10    /**< data gap encountered */
 #define STREAM_DEPTH            0x20    /**< depth reached */
 
-typedef struct StreamMsg_ {
-    struct StreamMsg_ *next;
-    struct StreamMsg_ *prev;
-
-    uint32_t seq;                   /**< sequence number */
-    uint32_t data_len;              /**< length of the data */
-    uint32_t data_size;
-    uint8_t *data;                  /**< reassembled data: ptr to after this
-                                     *   struct */
-} StreamMsg;
-
-typedef struct StreamMsgQueue_ {
-    StreamMsg *top;
-    StreamMsg *bot;
-    uint16_t len;
-#ifdef DBG_PERF
-    uint16_t dbg_maxlen;
-#endif /* DBG_PERF */
-} StreamMsgQueue;
-
-/* prototypes */
-void StreamMsgQueuesInit(uint32_t prealloc);
-void StreamMsgQueuesDeinit(char);
-
-StreamMsg *StreamMsgGetFromPool(void);
-void StreamMsgReturnToPool(StreamMsg *);
-StreamMsg *StreamMsgGetFromQueue(StreamMsgQueue *);
-void StreamMsgPutInQueue(StreamMsgQueue *, StreamMsg *);
-
-StreamMsgQueue *StreamMsgQueueGetNew(void);
-void StreamMsgQueueFree(StreamMsgQueue *);
-
-void StreamMsgQueueSetMinChunkLen(uint8_t dir, uint16_t len);
-uint16_t StreamMsgQueueGetMinChunkLen(uint8_t);
-
-void StreamMsgReturnListToPool(void *);
-
 typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,
                          StreamSegmentCallback CallbackFunc,
diff --git a/src/suricata.h b/src/suricata.h
index c49b7f43a0..c93036a234 100644
--- a/src/suricata.h
+++ b/src/suricata.h
@@ -47,7 +47,6 @@
  *  - ::Packet: Data relative to an individual packet with information about
  *  linked structure such as the ::Flow the ::Packet belongs to.
  *  - ::Flow: Information about a flow for example a TCP session
- *  - ::StreamMsg: structure containing the reassembled data
  *
  *  \subsection runmode Running mode
  *
diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index 6ab2e883fb..ff2728911b 100644
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -1336,7 +1336,6 @@ void CudaReleasePacket(Packet *p)
  *   tables on priority.
  * - Introduce profiling.
  * - Retrieve sgh before buffer packet.
- * - Buffer smsgs too.
  */
 
 void SCACConstructBoth16and32StateTables(void)