From: Victor Julien <victor@inliniac.net>
Date: Sun, 8 Oct 2017 09:42:30 +0000 (+0200)
Subject: detect: rewrite of the detect engine
X-Git-Tag: suricata-4.1.0-beta1~321
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=af51e0f5a12865a1d805f7010ee02e8b7444aa38;p=thirdparty%2Fsuricata.git

detect: rewrite of the detect engine

Use per tx detect_flags to track prefilter. Detect flags are used for 2
things:
1. marking tx as fully inspected
2. tracking already run prefilter (incl mpm) engines

This supercedes the MpmIDs API for directionless tracking
of the prefilter engines.

When we have no SGH we have to flag the txs that are 'complete'
as inspected as well.

Special handling for the stream engine:

If a rule mixes TX inspection and STREAM inspection, we can encounter
the case where the rule is evaluated against multiple transactions
during a single inspection run. As the stream data is exactly the same
for each of those runs, it's wasteful to rerun inspection of the stream
portion of the rule.

This patch enables caching of the stream 'inspect engine' result in
the local 'RuleMatchCandidateTx' array. This is valid only during the
live of a single inspection run.

Remove stateful inspection from 'mask' (SignatureMask). The mask wasn't
used in most cases for those rules anyway, as there we rely on the
prefilter. Add a alproto check to catch the remaining cases.

When building the active non-mpm/non-prefilter list check not just
the mask, but also the alproto. This especially helps stateful rules
with negated mpm.

Simplify AppLayerParserHasDecoderEvents usage in detection to only
return true if protocol detection events are set. Other detection is done
in inspect engines.

Move rule group lookup and handling into it's own function. Handle
'post lookup' tasks immediately, instead of after the first detect
run. The tasks were independent of the initial detection.

Many cleanups and much refactoring.
---

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 37a62d88fb..027706bcc4 100644
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -678,7 +678,8 @@ uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint
 }
 
 void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *pstate,
-                                           void *alstate, const uint8_t flags)
+                                           void *alstate, const uint8_t flags,
+                                           bool tag_txs_as_inspected)
 {
     SCEnter();
 
@@ -686,19 +687,49 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
     const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
     uint64_t idx = AppLayerParserGetTransactionInspectId(pstate, flags);
     const int state_done_progress = AppLayerParserGetStateProgressCompletionStatus(f->alproto, flags);
+    const uint8_t ipproto = f->proto;
+    const AppProto alproto = f->alproto;
 
     for (; idx < total_txs; idx++) {
-        void *tx = AppLayerParserGetTx(f->proto, f->alproto, alstate, idx);
+        void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
         if (tx == NULL)
             continue;
-        int state_progress = AppLayerParserGetStateProgress(f->proto, f->alproto, tx, flags);
-        if (state_progress >= state_done_progress)
+        int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
+        if (state_progress >= state_done_progress) {
+            if (tag_txs_as_inspected) {
+                uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+                if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                    detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                    AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                    SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64,
+                            tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
+                }
+            }
             continue;
-        else
+        } else
             break;
     }
     pstate->inspect_id[direction] = idx;
 
+    /* if necessary we flag all txs that are complete as 'inspected' */
+    if (tag_txs_as_inspected) {
+        for (; idx < total_txs; idx++) {
+            void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
+            if (tx == NULL)
+                continue;
+            int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
+            if (state_progress >= state_done_progress) {
+                uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+                if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                    detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                    AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                    SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64,
+                            tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
+                }
+            }
+        }
+    }
+
     SCReturn;
 }
 
@@ -1174,48 +1205,21 @@ void AppLayerParserSetEOF(AppLayerParserState *pstate)
     SCReturn;
 }
 
-bool AppLayerParserHasDecoderEvents(const Flow *f,
-        void *alstate, AppLayerParserState *pstate,
-        const uint8_t flags)
+/* return true if there are app parser decoder events. These are
+ * only the ones that are set during protocol detection. */
+bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate)
 {
     SCEnter();
 
-    if (alstate == NULL || pstate == NULL)
-        goto not_present;
-
-    AppLayerDecoderEvents *decoder_events;
-    uint64_t tx_id;
-    uint64_t max_id;
-
-    if (AppLayerParserProtocolIsTxEventAware(f->proto, f->alproto)) {
-        /* fast path if supported by alproto */
-        if (alp_ctx.ctxs[f->protomap][f->alproto].StateHasEvents != NULL) {
-            if (alp_ctx.ctxs[f->protomap][f->alproto].
-                StateHasEvents(alstate) == 1)
-            {
-                goto present;
-            }
-        } else {
-            /* check each tx */
-            tx_id = AppLayerParserGetTransactionInspectId(pstate, flags);
-            max_id = AppLayerParserGetTxCnt(f, alstate);
-            for ( ; tx_id < max_id; tx_id++) {
-                decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, alstate, tx_id);
-                if (decoder_events && decoder_events->cnt)
-                    goto present;
-            }
-        }
-    }
+    if (pstate == NULL)
+        return false;
 
-    decoder_events = AppLayerParserGetDecoderEvents(pstate);
+    const AppLayerDecoderEvents *decoder_events = AppLayerParserGetDecoderEvents(pstate);
     if (decoder_events && decoder_events->cnt)
-        goto present;
+        return true;
 
     /* if we have reached here, we don't have events */
- not_present:
     return false;
- present:
-    return true;
 }
 
 /** \brief simpler way to globally test if a alproto is registered
diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index 4a85e02271..b40eecf47c 100644
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -195,7 +195,7 @@ LoggerId AppLayerParserGetTxLogged(const Flow *f, void *alstate, void *tx);
 
 uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState *pstate, uint8_t direction);
 void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *pstate,
-                                void *alstate, const uint8_t flags);
+                                void *alstate, const uint8_t flags, bool tag_txs_as_inspected);
 
 AppLayerDecoderEvents *AppLayerParserGetDecoderEvents(AppLayerParserState *pstate);
 void AppLayerParserSetDecoderEvents(AppLayerParserState *pstate, AppLayerDecoderEvents *devents);
@@ -232,8 +232,7 @@ int AppLayerParserSetTxMpmIDs(uint8_t ipproto, AppProto alproto, void *tx, uint6
 int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *tctx, Flow *f, AppProto alproto,
                    uint8_t flags, uint8_t *input, uint32_t input_len);
 void AppLayerParserSetEOF(AppLayerParserState *pstate);
-bool AppLayerParserHasDecoderEvents(const Flow *f, void *alstate, AppLayerParserState *pstate,
-                        const uint8_t flags);
+bool AppLayerParserHasDecoderEvents(AppLayerParserState *pstate);
 int AppLayerParserIsTxAware(AppProto alproto);
 int AppLayerParserProtocolIsTxAware(uint8_t ipproto, AppProto alproto);
 int AppLayerParserProtocolIsTxEventAware(uint8_t ipproto, AppProto alproto);
diff --git a/src/app-layer-protos.h b/src/app-layer-protos.h
index 49d1c8cef2..0df2c80cc1 100644
--- a/src/app-layer-protos.h
+++ b/src/app-layer-protos.h
@@ -58,6 +58,7 @@ enum AppProtoEnum {
     /* keep last */
     ALPROTO_MAX,
 };
+// NOTE: if ALPROTO's get >= 256, update SignatureNonPrefilterStore
 
 /* not using the enum as that is a unsigned int, so 4 bytes */
 typedef uint16_t AppProto;
diff --git a/src/detect-app-layer-event.c b/src/detect-app-layer-event.c
index 02e58560a1..be2bb5aedc 100644
--- a/src/detect-app-layer-event.c
+++ b/src/detect-app-layer-event.c
@@ -58,7 +58,6 @@ static int DetectEngineAptEventInspect(ThreadVars *tv,
         const Signature *s, const SigMatchData *smd,
         Flow *f, uint8_t flags, void *alstate,
         void *tx, uint64_t tx_id);
-static void DetectAppLayerEventSetupCallback(Signature *s);
 static int g_applayer_events_list_id = 0;
 
 /**
@@ -81,9 +80,6 @@ void DetectAppLayerEventRegister(void)
             ALPROTO_UNKNOWN, SIG_FLAG_TOCLIENT, 0,
             DetectEngineAptEventInspect);
 
-    DetectBufferTypeRegisterSetupCallback("app-layer-events",
-            DetectAppLayerEventSetupCallback);
-
     g_applayer_events_list_id = DetectBufferTypeGetByName("app-layer-events");
 }
 
@@ -146,38 +142,6 @@ static int DetectAppLayerEventPktMatch(ThreadVars *t, DetectEngineThreadCtx *det
                                            aled->event_id);
 }
 
-static void DetectAppLayerEventSetupCallback(Signature *s)
-{
-    SigMatch *sm;
-    for (sm = s->init_data->smlists[g_applayer_events_list_id] ; sm != NULL; sm = sm->next) {
-        switch (sm->type) {
-            case DETECT_AL_APP_LAYER_EVENT:
-            {
-                DetectAppLayerEventData *aed = (DetectAppLayerEventData *)sm->ctx;
-                switch (aed->alproto) {
-                    case ALPROTO_HTTP:
-                        s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-                        SCLogDebug("sig %u requires http app state (http event)", s->id);
-                        break;
-                    case ALPROTO_SMTP:
-                        s->mask |= SIG_MASK_REQUIRE_SMTP_STATE;
-                        SCLogDebug("sig %u requires smtp app state (smtp event)", s->id);
-                        break;
-                    case ALPROTO_DNS:
-                        s->mask |= SIG_MASK_REQUIRE_DNS_STATE;
-                        SCLogDebug("sig %u requires dns app state (dns event)", s->id);
-                        break;
-                    case ALPROTO_TLS:
-                        s->mask |= SIG_MASK_REQUIRE_TLS_STATE;
-                        SCLogDebug("sig %u requires tls app state (tls event)", s->id);
-                        break;
-                }
-                break;
-            }
-        }
-    }
-}
-
 static DetectAppLayerEventData *DetectAppLayerEventParsePkt(const char *arg,
                                                             AppLayerEventType *event_type)
 {
diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c
index 329d43c557..9fa32c2f6e 100644
--- a/src/detect-engine-build.c
+++ b/src/detect-engine-build.c
@@ -403,7 +403,7 @@ deonly:
  */
 void
 PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
-        bool has_state, bool app_decoder_events)
+        bool app_decoder_events)
 {
     if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
         SCLogDebug("packet has payload");
@@ -433,62 +433,6 @@ PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
     if (p->flags & PKT_HAS_FLOW) {
         SCLogDebug("packet has flow");
         (*mask) |= SIG_MASK_REQUIRE_FLOW;
-
-        if (has_state) {
-            switch(alproto) {
-                case ALPROTO_HTTP:
-                    SCLogDebug("packet/flow has http state");
-                    (*mask) |= SIG_MASK_REQUIRE_HTTP_STATE;
-                    break;
-                case ALPROTO_SMB:
-                case ALPROTO_SMB2:
-                case ALPROTO_DCERPC:
-                    SCLogDebug("packet/flow has dce state");
-                    (*mask) |= SIG_MASK_REQUIRE_DCE_STATE;
-                    break;
-                case ALPROTO_SSH:
-                    SCLogDebug("packet/flow has ssh state");
-                    (*mask) |= SIG_MASK_REQUIRE_SSH_STATE;
-                    break;
-                case ALPROTO_TLS:
-                    SCLogDebug("packet/flow has tls state");
-                    (*mask) |= SIG_MASK_REQUIRE_TLS_STATE;
-                    break;
-                case ALPROTO_DNS:
-                    SCLogDebug("packet/flow has dns state");
-                    (*mask) |= SIG_MASK_REQUIRE_DNS_STATE;
-                    break;
-                case ALPROTO_FTP:
-                    SCLogDebug("packet/flow has ftp state");
-                    (*mask) |= SIG_MASK_REQUIRE_FTP_STATE;
-                    break;
-                case ALPROTO_FTPDATA:
-                    SCLogDebug("packet/flow has ftpdata state");
-                    (*mask) |= SIG_MASK_REQUIRE_FTPDATA_STATE;
-                    break;
-                case ALPROTO_SMTP:
-                    SCLogDebug("packet/flow has smtp state");
-                    (*mask) |= SIG_MASK_REQUIRE_SMTP_STATE;
-                    break;
-                case ALPROTO_ENIP:
-                    SCLogDebug("packet/flow has enip state");
-                    (*mask) |= SIG_MASK_REQUIRE_ENIP_STATE;
-                    break;
-                case ALPROTO_DNP3:
-                    SCLogDebug("packet/flow has dnp3 state");
-                    (*mask) |= SIG_MASK_REQUIRE_DNP3_STATE;
-                    break;
-                case ALPROTO_TEMPLATE:
-                    SCLogDebug("packet/flow has template state");
-                    (*mask) |= SIG_MASK_REQUIRE_TEMPLATE_STATE;
-                    break;
-                default:
-                    SCLogDebug("packet/flow has other state");
-                    break;
-            }
-        } else {
-            SCLogDebug("no alstate");
-        }
     }
 }
 
@@ -593,59 +537,6 @@ static int SignatureCreateMask(Signature *s)
         }
     }
 
-    if (s->alproto == ALPROTO_SSH) {
-        s->mask |= SIG_MASK_REQUIRE_SSH_STATE;
-        SCLogDebug("sig requires ssh state");
-    }
-    if (s->alproto == ALPROTO_TLS) {
-        s->mask |= SIG_MASK_REQUIRE_TLS_STATE;
-        SCLogDebug("sig requires tls state");
-    }
-    if (s->alproto == ALPROTO_DNS) {
-        s->mask |= SIG_MASK_REQUIRE_DNS_STATE;
-        SCLogDebug("sig requires dns state");
-    }
-    if (s->alproto == ALPROTO_DNP3) {
-        s->mask |= SIG_MASK_REQUIRE_DNP3_STATE;
-        SCLogDebug("sig requires dnp3 state");
-    }
-    if (s->alproto == ALPROTO_FTP) {
-        s->mask |= SIG_MASK_REQUIRE_FTP_STATE;
-        SCLogDebug("sig requires ftp state");
-    }
-    if (s->alproto == ALPROTO_FTPDATA) {
-        s->mask |= SIG_MASK_REQUIRE_FTPDATA_STATE;
-        SCLogDebug("sig requires ftp data state");
-    }
-    if (s->alproto == ALPROTO_SMTP) {
-        s->mask |= SIG_MASK_REQUIRE_SMTP_STATE;
-        SCLogDebug("sig requires smtp state");
-    }
-    if (s->alproto == ALPROTO_ENIP) {
-        s->mask |= SIG_MASK_REQUIRE_ENIP_STATE;
-        SCLogDebug("sig requires enip state");
-    }
-    if (s->alproto == ALPROTO_TEMPLATE) {
-        s->mask |= SIG_MASK_REQUIRE_TEMPLATE_STATE;
-        SCLogDebug("sig requires template state");
-    }
-
-    if ((s->mask & SIG_MASK_REQUIRE_DCE_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_HTTP_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_SSH_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_DNS_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_DNP3_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_FTP_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_FTPDATA_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_SMTP_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_ENIP_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_TEMPLATE_STATE) ||
-        (s->mask & SIG_MASK_REQUIRE_TLS_STATE))
-    {
-        s->mask |= SIG_MASK_REQUIRE_FLOW;
-        SCLogDebug("sig requires flow");
-    }
-
     if (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) {
         s->mask |= SIG_MASK_REQUIRE_FLOW;
         SCLogDebug("sig requires flow");
diff --git a/src/detect-engine-build.h b/src/detect-engine-build.h
index 25322f6410..8775ee1191 100644
--- a/src/detect-engine-build.h
+++ b/src/detect-engine-build.h
@@ -19,7 +19,7 @@
 #define __DETECT_ENGINE_BUILD_H__
 
 void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
-        bool has_state, bool app_decoder_events);
+        bool app_decoder_events);
 
 int SignatureIsFilestoring(const Signature *);
 int SignatureIsFilemagicInspecting(const Signature *);
diff --git a/src/detect-engine-iponly.c b/src/detect-engine-iponly.c
index af19ebe69a..4fc440d6a7 100644
--- a/src/detect-engine-iponly.c
+++ b/src/detect-engine-iponly.c
@@ -974,9 +974,9 @@ int IPOnlyMatchCompatSMs(ThreadVars *tv,
  * \param p Pointer to the Packet to match against
  */
 void IPOnlyMatchPacket(ThreadVars *tv,
-                       DetectEngineCtx *de_ctx,
+                       const DetectEngineCtx *de_ctx,
                        DetectEngineThreadCtx *det_ctx,
-                       DetectEngineIPOnlyCtx *io_ctx,
+                       const DetectEngineIPOnlyCtx *io_ctx,
                        DetectEngineIPOnlyThreadCtx *io_tctx, Packet *p)
 {
     SigNumArray *src = NULL;
diff --git a/src/detect-engine-iponly.h b/src/detect-engine-iponly.h
index b71a59337d..155459ddd8 100644
--- a/src/detect-engine-iponly.h
+++ b/src/detect-engine-iponly.h
@@ -37,8 +37,8 @@ typedef struct SigNumArray_ {
 
 void IPOnlyCIDRListFree(IPOnlyCIDRItem *tmphead);
 int IPOnlySigParseAddress(const DetectEngineCtx *, Signature *, const char *, char);
-void IPOnlyMatchPacket(ThreadVars *tv, DetectEngineCtx *,
-                       DetectEngineThreadCtx *, DetectEngineIPOnlyCtx *,
+void IPOnlyMatchPacket(ThreadVars *tv, const DetectEngineCtx *,
+                       DetectEngineThreadCtx *, const DetectEngineIPOnlyCtx *,
                        DetectEngineIPOnlyThreadCtx *, Packet *);
 void IPOnlyInit(DetectEngineCtx *, DetectEngineIPOnlyCtx *);
 void IPOnlyPrint(DetectEngineCtx *, DetectEngineIPOnlyCtx *);
diff --git a/src/detect-engine-payload.c b/src/detect-engine-payload.c
index 313c897544..b648db9fff 100644
--- a/src/detect-engine-payload.c
+++ b/src/detect-engine-payload.c
@@ -326,9 +326,6 @@ int DetectEngineInspectStream(ThreadVars *tv,
     if (ssn == NULL)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
 
-    if (det_ctx->stream_already_inspected)
-        return det_ctx->stream_last_result;
-
     uint64_t unused;
     struct StreamContentInspectEngineData inspect_data = { de_ctx, det_ctx, s, smd, f };
     int match = StreamReassembleRaw(f->protoctx, p,
@@ -349,19 +346,15 @@ int DetectEngineInspectStream(ThreadVars *tv,
     SCLogDebug("%s ran stream for sid %u on packet %"PRIu64" and we %s",
             is_last? "LAST:" : "normal:", s->id, p->pcap_cnt,
             match ? "matched" : "didn't match");
-    det_ctx->stream_already_inspected = true;
 
     if (match) {
-        det_ctx->stream_last_result = DETECT_ENGINE_INSPECT_SIG_MATCH;
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
     } else {
         if (is_last) {
-            det_ctx->stream_last_result = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
             //SCLogNotice("last, so DETECT_ENGINE_INSPECT_SIG_CANT_MATCH");
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         }
         /* TODO maybe we can set 'CANT_MATCH' for EOF too? */
-        det_ctx->stream_last_result = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 }
diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c
index 5478902cf4..380ea87568 100644
--- a/src/detect-engine-prefilter.c
+++ b/src/detect-engine-prefilter.c
@@ -88,75 +88,60 @@ static inline void QuickSortSigIntId(SigIntId *sids, uint32_t n)
     QuickSortSigIntId(l, sids + n - l);
 }
 
-static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx,
-        const SigGroupHead *sgh, Packet *p, const uint8_t flags)
+/**
+ * \brief run prefilter engines on a transaction
+ */
+void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
+        const SigGroupHead *sgh,
+        Packet *p,
+        const uint8_t ipproto,
+        const uint8_t flow_flags,
+        const AppProto alproto,
+        void *alstate,
+        DetectTransaction *tx)
 {
-    SCEnter();
-
-    const AppProto alproto = p->flow->alproto;
-    const uint8_t ipproto = p->proto;
-
-    if (!(AppLayerParserProtocolIsTxAware(ipproto, alproto)))
-        SCReturn;
-
-    void *alstate = p->flow->alstate;
-    uint64_t idx = AppLayerParserGetTransactionInspectId(p->flow->alparser, flags);
-    const uint64_t total_txs = AppLayerParserGetTxCnt(p->flow, alstate);
-
-    /* HACK test HTTP state here instead of in each engine */
-    if (alproto == ALPROTO_HTTP) {
-        HtpState *htp_state = (HtpState *)alstate;
-        if (unlikely(htp_state->connp == NULL)) {
-            SCLogDebug("no HTTP connp");
-            SCReturn;
-        }
-    }
-
-    /* run our engines against each tx */
-    for (; idx < total_txs; idx++) {
-        void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
-        if (tx == NULL)
-            continue;
-
-        uint64_t mpm_ids = AppLayerParserGetTxMpmIDs(ipproto, alproto, tx);
-        const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
-        SCLogDebug("tx %p progress %d", tx, tx_progress);
-
-        PrefilterEngine *engine = sgh->tx_engines;
-        do {
-            if (engine->alproto != alproto)
-                goto next;
-            if (engine->tx_min_progress > tx_progress)
+    /* reset rule store */
+    det_ctx->pmq.rule_id_array_cnt = 0;
+
+    SCLogDebug("tx %p progress %d", tx->tx_ptr, tx->tx_progress);
+
+    PrefilterEngine *engine = sgh->tx_engines;
+    do {
+        if (engine->alproto != alproto)
+            goto next;
+        if (engine->tx_min_progress > tx->tx_progress)
+            goto next;
+        if (tx->tx_progress > engine->tx_min_progress) {
+            if (tx->prefilter_flags & (1<<(engine->id))) {
                 goto next;
-            if (tx_progress > engine->tx_min_progress) {
-                if (mpm_ids & (1<<(engine->gid))) {
-                    goto next;
-                }
             }
+        }
 
-            PROFILING_PREFILTER_START(p);
-            engine->cb.PrefilterTx(det_ctx, engine->pectx,
-                    p, p->flow, tx, idx, flags);
-            PROFILING_PREFILTER_END(p, engine->gid);
+        PROFILING_PREFILTER_START(p);
+        engine->cb.PrefilterTx(det_ctx, engine->pectx,
+                p, p->flow, tx->tx_ptr, tx->tx_id, flow_flags);
+        PROFILING_PREFILTER_END(p, engine->gid);
 
-            if (tx_progress > engine->tx_min_progress) {
-                mpm_ids |= (1<<(engine->gid));
-            }
-        next:
-            if (engine->is_last)
-                break;
-            engine++;
-        } while (1);
-
-        if (mpm_ids != 0) {
-            //SCLogNotice("tx %p Mpm IDs: %"PRIx64, tx, mpm_ids);
-            AppLayerParserSetTxMpmIDs(ipproto, alproto, tx, mpm_ids);
+        if (tx->tx_progress > engine->tx_min_progress) {
+            tx->prefilter_flags |= (1<<(engine->id));
         }
+    next:
+        if (engine->is_last)
+            break;
+        engine++;
+    } while (1);
+
+    /* Sort the rule list to lets look at pmq.
+     * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */
+    if (likely(det_ctx->pmq.rule_id_array_cnt > 1)) {
+        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT1);
+        QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt);
+        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT1);
     }
 }
 
 void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
-        Packet *p, const uint8_t flags, const bool has_state)
+        Packet *p, const uint8_t flags)
 {
     SCEnter();
 
@@ -197,17 +182,6 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
         PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_PAYLOAD);
     }
 
-    /* run tx engines */
-    if (((p->proto == IPPROTO_TCP && p->flowflags & FLOW_PKT_ESTABLISHED) || p->proto != IPPROTO_TCP) && has_state) {
-        if (sgh->tx_engines != NULL && p->flow != NULL &&
-                p->flow->alproto != ALPROTO_UNKNOWN && p->flow->alstate != NULL)
-        {
-            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_TX);
-            PrefilterTx(det_ctx, sgh, p, flags);
-            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_TX);
-        }
-    }
-
     /* Sort the rule list to lets look at pmq.
      * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */
     if (likely(det_ctx->pmq.rule_id_array_cnt > 1)) {
diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h
index 2c49136d9a..1a5f03b819 100644
--- a/src/detect-engine-prefilter.h
+++ b/src/detect-engine-prefilter.h
@@ -24,8 +24,10 @@
 #ifndef __DETECT_ENGINE_PREFILTER_H__
 #define __DETECT_ENGINE_PREFILTER_H__
 
+#include "detect-engine-state.h"
+
 void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p,
-        const uint8_t flags, const bool has_state);
+        const uint8_t flags);
 
 int PrefilterAppendEngine(SigGroupHead *sgh,
         void (*Prefilter)(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx),
@@ -43,6 +45,15 @@ int PrefilterAppendTxEngine(SigGroupHead *sgh,
         void *pectx, void (*FreeFunc)(void *pectx),
         const char *name);
 
+void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
+        const SigGroupHead *sgh,
+        Packet *p,
+        const uint8_t ipproto,
+        const uint8_t flow_flags,
+        const AppProto alproto,
+        void *alstate,
+        DetectTransaction *tx);
+
 void PrefilterFreeEnginesList(PrefilterEngineList *list);
 
 void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
diff --git a/src/detect-engine-profile.c b/src/detect-engine-profile.c
index 24cc48467d..fc43c3fd01 100644
--- a/src/detect-engine-profile.c
+++ b/src/detect-engine-profile.c
@@ -57,7 +57,8 @@ static void DumpFp(const SigMatch *sm, char *pat_orig, uint32_t pat_orig_sz, cha
 #endif
 
 SCMutex g_rule_dump_write_m = SCMUTEX_INITIALIZER;
-void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p)
+void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx,
+        const SigGroupHead *sgh, const Packet *p)
 {
     json_t *js = CreateJSONHeader(p, 0, "inspectedrules");
     if (js == NULL)
@@ -66,7 +67,7 @@ void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p)
     if (ir == NULL)
         return;
 
-    json_object_set_new(ir, "rule_group_id", json_integer(det_ctx->sgh->id));
+    json_object_set_new(ir, "rule_group_id", json_integer(sgh->id));
     json_object_set_new(ir, "rule_cnt", json_integer(det_ctx->match_array_cnt));
 
     json_t *js_array = json_array();
diff --git a/src/detect-engine-profile.h b/src/detect-engine-profile.h
index d7ed1c93ea..55d64d1dda 100644
--- a/src/detect-engine-profile.h
+++ b/src/detect-engine-profile.h
@@ -24,6 +24,7 @@
 #ifndef _DETECT_ENGINE_PROFILE_H
 #define	_DETECT_ENGINE_PROFILE_H
 
-void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p);
+void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx,
+        const SigGroupHead *sgh, const Packet *p);
 
 #endif	/* _DETECT_ENGINE_PROFILE_H */
diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index a845a0c514..3a839dc4c1 100644
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -696,6 +696,7 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg
                 BUG_ON(sgh->non_pf_other_store_array == NULL);
                 sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].id = s->num;
                 sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].mask = s->mask;
+                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].alproto = s->alproto;
                 sgh->non_pf_other_store_cnt++;
             }
 
@@ -703,6 +704,7 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg
             BUG_ON(sgh->non_pf_syn_store_array == NULL);
             sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].id = s->num;
             sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].mask = s->mask;
+            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].alproto = s->alproto;
             sgh->non_pf_syn_store_cnt++;
         }
     }
diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c
index 2d9559dbdc..9cc5ff2865 100644
--- a/src/detect-engine-state.c
+++ b/src/detect-engine-state.c
@@ -81,6 +81,7 @@
 /** convert enum to string */
 #define CASE_CODE(E)  case E: return #E
 
+#if 0
 /** The DetectEngineThreadCtx::de_state_sig_array contains 2 separate values:
  *  1. the first bit tells the prefilter engine to bypass the rule (or not)
  *  2. the other bits allow 'ContinueDetect' to specify an offset again the
@@ -94,6 +95,7 @@
 #define MAX_STORED_TXID_OFFSET 127
 
 /******** static internal helpers *********/
+#endif
 
 static inline int StateIsValid(uint16_t alproto, void *alstate)
 {
@@ -110,12 +112,14 @@ static inline int StateIsValid(uint16_t alproto, void *alstate)
     return 0;
 }
 
+#if 0
 static inline int TxIsLast(uint64_t tx_id, uint64_t total_txs)
 {
     if (total_txs - tx_id <= 1)
         return 1;
     return 0;
 }
+#endif
 
 static DeStateStore *DeStateStoreAlloc(void)
 {
@@ -127,6 +131,7 @@ static DeStateStore *DeStateStoreAlloc(void)
     return d;
 }
 
+#ifdef DEBUG_VALIDATION
 static int DeStateSearchState(DetectEngineState *state, uint8_t direction, SigIntId num)
 {
     DetectEngineStateDirection *dir_state = &state->dir_state[direction & STREAM_TOSERVER ? 0 : 1];
@@ -151,6 +156,7 @@ static int DeStateSearchState(DetectEngineState *state, uint8_t direction, SigIn
     }
     return 0;
 }
+#endif
 
 static void DeStateSignatureAppend(DetectEngineState *state,
         const Signature *s, uint32_t inspect_flags, uint8_t direction)
@@ -194,21 +200,6 @@ static void DeStateSignatureAppend(DetectEngineState *state,
     return;
 }
 
-static void DeStateStoreFileNoMatchCnt(DetectEngineState *de_state, uint16_t file_no_match, uint8_t direction)
-{
-    de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].filestore_cnt += file_no_match;
-
-    return;
-}
-
-static int DeStateStoreFilestoreSigsCantMatch(const SigGroupHead *sgh, DetectEngineState *de_state, uint8_t direction)
-{
-    if (de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].filestore_cnt == sgh->filestore_cnt)
-        return 1;
-    else
-        return 0;
-}
-
 DetectEngineState *DetectEngineStateAlloc(void)
 {
     DetectEngineState *d = SCMalloc(sizeof(DetectEngineState));
@@ -238,75 +229,37 @@ void DetectEngineStateFree(DetectEngineState *state)
     return;
 }
 
-static int HasStoredSigs(const Flow *f, const uint8_t flags)
+static void StoreFileNoMatchCnt(DetectEngineState *de_state, uint16_t file_no_match, uint8_t direction)
 {
-    AppProto alproto = f->alproto;
-    void *alstate = FlowGetAppState(f);
-    if (!StateIsValid(f->alproto, alstate)) {
-        return 0;
-    }
-
-    int state = AppLayerParserHasTxDetectState(f->proto, alproto, f->alstate);
-    if (state == -ENOSYS) { /* proto doesn't support this API call */
-        /* fall through */
-    } else if (state == 0) {
-        return 0;
-    }
-    /* if state == 1 we also fall through */
-
-    uint64_t inspect_tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flags);
-    uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
+    de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].filestore_cnt += file_no_match;
 
-    for ( ; inspect_tx_id < total_txs; inspect_tx_id++) {
-        void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id);
-        if (inspect_tx != NULL) {
-            DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(f->proto, alproto, inspect_tx);
-            if (tx_de_state == NULL) {
-                continue;
-            }
-            if (tx_de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].cnt != 0) {
-                SCLogDebug("tx %"PRIu64" has sigs present", inspect_tx_id);
-                return 1;
-            }
-        }
-    }
-    return 0;
+    return;
 }
 
-/** \brief Check if we need to inspect this state
- *
- *  State needs to be inspected if:
- *   1. state has been updated
- *   2. we already have de_state in progress
- *
- *  \retval 0 no inspectable state
- *  \retval 1 inspectable state
- */
-int DeStateFlowHasInspectableState(const Flow *f, const uint8_t flags)
+static bool StoreFilestoreSigsCantMatch(const SigGroupHead *sgh, const DetectEngineState *de_state, uint8_t direction)
 {
-    int r = 0;
-
-    if (HasStoredSigs(f, flags)) {
-        r = 1;
-    } else {
-        r = 0;
-    }
-    return r;
+    if (de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].filestore_cnt == sgh->filestore_cnt)
+        return true;
+    else
+        return false;
 }
 
-static void StoreStateTxHandleFiles(DetectEngineThreadCtx *det_ctx, Flow *f,
-                                    DetectEngineState *destate, const uint8_t flags,
+static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f,
+                                    DetectEngineState *destate, const uint8_t flow_flags,
                                     const uint64_t tx_id, const uint16_t file_no_match)
 {
     SCLogDebug("tx %"PRIu64", file_no_match %u", tx_id, file_no_match);
-    DeStateStoreFileNoMatchCnt(destate, file_no_match, flags);
-    if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, destate, flags) == 1) {
-        FileDisableStoringForTransaction(f, flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id);
+    StoreFileNoMatchCnt(destate, file_no_match, flow_flags);
+    if (StoreFilestoreSigsCantMatch(sgh, destate, flow_flags)) {
+        FileDisableStoringForTransaction(f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id);
     }
 }
 
-static void StoreStateTxFileOnly(DetectEngineThreadCtx *det_ctx,
-        Flow *f, const uint8_t flags, const uint64_t tx_id, void *tx,
+void DetectRunStoreStateTx(
+        const SigGroupHead *sgh,
+        Flow *f, void *tx, uint64_t tx_id,
+        const Signature *s,
+        uint32_t inspect_flags, uint8_t flow_flags,
         const uint16_t file_no_match)
 {
     DetectEngineState *destate = AppLayerParserGetTxDetectState(f->proto, f->alproto, tx);
@@ -320,9 +273,33 @@ static void StoreStateTxFileOnly(DetectEngineThreadCtx *det_ctx,
         }
         SCLogDebug("destate created for %"PRIu64, tx_id);
     }
-    StoreStateTxHandleFiles(det_ctx, f, destate, flags, tx_id, file_no_match);
+    DeStateSignatureAppend(destate, s, inspect_flags, flow_flags);
+    StoreStateTxHandleFiles(sgh, f, destate, flow_flags, tx_id, file_no_match);
+
+    SCLogDebug("Stored for TX %"PRIu64, tx_id);
 }
 
+void DetectRunStoreStateTxFileOnly(
+        const SigGroupHead *sgh,
+        Flow *f, void *tx, uint64_t tx_id,
+        const uint8_t flow_flags,
+        const uint16_t file_no_match)
+{
+    DetectEngineState *destate = AppLayerParserGetTxDetectState(f->proto, f->alproto, tx);
+    if (destate == NULL) {
+        destate = DetectEngineStateAlloc();
+        if (destate == NULL)
+            return;
+        if (AppLayerParserSetTxDetectState(f, f->alstate, tx, destate) < 0) {
+            DetectEngineStateFree(destate);
+            return;
+        }
+        SCLogDebug("destate created for %"PRIu64, tx_id);
+    }
+    StoreStateTxHandleFiles(sgh, f, destate, flow_flags, tx_id, file_no_match);
+}
+
+#if 0
 /**
  *  \param check_before_add check for duplicates before adding the sig
  */
@@ -353,6 +330,62 @@ static void StoreStateTx(DetectEngineThreadCtx *det_ctx,
     SCLogDebug("Stored for TX %"PRIu64, tx_id);
 }
 
+static int HasStoredSigs(const Flow *f, const uint8_t flags)
+{
+    AppProto alproto = f->alproto;
+    void *alstate = FlowGetAppState(f);
+    if (!StateIsValid(f->alproto, alstate)) {
+        return 0;
+    }
+
+    int state = AppLayerParserHasTxDetectState(f->proto, alproto, f->alstate);
+    if (state == -ENOSYS) { /* proto doesn't support this API call */
+        /* fall through */
+    } else if (state == 0) {
+        return 0;
+    }
+    /* if state == 1 we also fall through */
+
+    uint64_t inspect_tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flags);
+    uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
+
+    for ( ; inspect_tx_id < total_txs; inspect_tx_id++) {
+        void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id);
+        if (inspect_tx != NULL) {
+            DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(f->proto, alproto, inspect_tx);
+            if (tx_de_state == NULL) {
+                continue;
+            }
+            if (tx_de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].cnt != 0) {
+                SCLogDebug("tx %"PRIu64" has sigs present", inspect_tx_id);
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+/** \brief Check if we need to inspect this state
+ *
+ *  State needs to be inspected if:
+ *   1. state has been updated
+ *   2. we already have de_state in progress
+ *
+ *  \retval 0 no inspectable state
+ *  \retval 1 inspectable state
+ */
+int DeStateFlowHasInspectableState(const Flow *f, const uint8_t flags)
+{
+    int r = 0;
+
+    if (HasStoredSigs(f, flags)) {
+        r = 1;
+    } else {
+        r = 0;
+    }
+    return r;
+}
+
 /* returns: true match, false no match */
 bool DeStateDetectStartDetection(ThreadVars *tv,
                                 DetectEngineCtx *de_ctx,
@@ -822,17 +855,23 @@ end:
     det_ctx->tx_id_set = 0;
     return;
 }
+#endif
+
 /** \brief update flow's inspection id's
  *
  *  \param f unlocked flow
  *  \param flags direction and disruption flags
+ *  \param tag_txs_as_inspected if true all 'complete' txs will be marked
+ *                              'inspected'
  *
  *  \note it is possible that f->alstate, f->alparser are NULL */
-void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags)
+void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags,
+        const bool tag_txs_as_inspected)
 {
     if (f->alparser && f->alstate) {
         AppLayerParserSetTransactionInspectId(f, f->alparser,
-                                              f->alstate, flags);
+                                              f->alstate, flags,
+                                              tag_txs_as_inspected);
     }
     return;
 }
diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h
index 860b0c8973..99631b77fc 100644
--- a/src/detect-engine-state.h
+++ b/src/detect-engine-state.h
@@ -82,8 +82,8 @@
  * the HAS_NEW_STATE flag, while if we don't have a new tx, we set
  * NO_NEW_STATE, to avoid getting the sig reinspected for the already
  * inspected tx. */
-#define DE_STATE_MATCH_HAS_NEW_STATE 0x00
-#define DE_STATE_MATCH_NO_NEW_STATE  0x80
+//#define DE_STATE_MATCH_HAS_NEW_STATE 0x00
+//#define DE_STATE_MATCH_NO_NEW_STATE  0x80
 
 typedef struct DeStateStoreItem_ {
     uint32_t flags;
@@ -108,6 +108,18 @@ typedef struct DetectEngineState_ {
     DetectEngineStateDirection dir_state[2];
 } DetectEngineState;
 
+// TODO
+typedef struct DetectTransaction_ {
+    void *tx_ptr;
+    const uint64_t tx_id;
+    DetectEngineStateDirection *de_state;
+    const uint64_t detect_flags;            /* detect flags get/set from/to applayer */
+    uint64_t prefilter_flags;               /* prefilter flags for direction, to be updated by prefilter code */
+    const uint64_t prefilter_flags_orig;    /* prefilter flags for direction, before prefilter has run */
+    const int tx_progress;
+    const int tx_end_state;
+} DetectTransaction;
+
 /**
  * \brief Alloc a DetectEngineState object.
  *
@@ -173,12 +185,27 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
  *  \param f unlocked flow
  *  \param flags direction and disruption flags
  */
-void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags);
+void DeStateUpdateInspectTransactionId(Flow *f, const uint8_t flags,
+        const bool tag_txs_as_inspected);
 
 void DetectEngineStateResetTxs(Flow *f);
 
 void DeStateRegisterTests(void);
 
+
+void DetectRunStoreStateTx(
+        const SigGroupHead *sgh,
+        Flow *f, void *tx, uint64_t tx_id,
+        const Signature *s,
+        uint32_t inspect_flags, uint8_t flow_flags,
+        const uint16_t file_no_match);
+
+void DetectRunStoreStateTxFileOnly(
+        const SigGroupHead *sgh,
+        Flow *f, void *tx, uint64_t tx_id,
+        const uint8_t flow_flags,
+        const uint16_t file_no_match);
+
 #endif /* __DETECT_ENGINE_STATE_H__ */
 
 /**
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 652329ea7d..b7d1385496 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -168,6 +168,7 @@ static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int di
     }
     new_engine->alproto = ALPROTO_UNKNOWN; /* all */
     new_engine->dir = direction;
+    new_engine->stream = true;
     new_engine->sm_list = DETECT_SM_LIST_PMATCH;
     new_engine->smd = stream;
     new_engine->Callback = DetectEngineInspectStream;
@@ -1738,14 +1739,6 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
 
     /* DeState */
     if (de_ctx->sig_array_len > 0) {
-        det_ctx->de_state_sig_array_len = de_ctx->sig_array_len;
-        det_ctx->de_state_sig_array = SCMalloc(det_ctx->de_state_sig_array_len * sizeof(uint8_t));
-        if (det_ctx->de_state_sig_array == NULL) {
-            return TM_ECODE_FAILED;
-        }
-        memset(det_ctx->de_state_sig_array, 0,
-               det_ctx->de_state_sig_array_len * sizeof(uint8_t));
-
         det_ctx->match_array_len = de_ctx->sig_array_len;
         det_ctx->match_array = SCMalloc(det_ctx->match_array_len * sizeof(Signature *));
         if (det_ctx->match_array == NULL) {
@@ -1753,6 +1746,8 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
         }
         memset(det_ctx->match_array, 0,
                det_ctx->match_array_len * sizeof(Signature *));
+
+        RuleMatchCandidateTxArrayInit(det_ctx, de_ctx->sig_array_len);
     }
 
     /* byte_extract storage */
@@ -1957,11 +1952,11 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     if (det_ctx->non_pf_id_array != NULL)
         SCFree(det_ctx->non_pf_id_array);
 
-    if (det_ctx->de_state_sig_array != NULL)
-        SCFree(det_ctx->de_state_sig_array);
     if (det_ctx->match_array != NULL)
         SCFree(det_ctx->match_array);
 
+    RuleMatchCandidateTxArrayFree(det_ctx);
+
     if (det_ctx->bj_values != NULL)
         SCFree(det_ctx->bj_values);
 
diff --git a/src/detect-file-data.c b/src/detect-file-data.c
index 4cb4c5dcba..c9754b9bfd 100644
--- a/src/detect-file-data.c
+++ b/src/detect-file-data.c
@@ -172,11 +172,6 @@ static void DetectFiledataSetupCallback(Signature *s)
     if (s->alproto == ALPROTO_HTTP || s->alproto == ALPROTO_UNKNOWN) {
         AppLayerHtpEnableRequestBodyCallback();
     }
-    if (s->alproto == ALPROTO_HTTP) {
-        s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-    } else if (s->alproto == ALPROTO_SMTP) {
-        s->mask |= SIG_MASK_REQUIRE_SMTP_STATE;
-    }
 
     SCLogDebug("callback invoked by %u", s->id);
 }
diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index cdbc204d30..aa8c87c280 100644
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -99,7 +99,6 @@ static void DetectHttpClientBodySetupCallback(Signature *s)
 {
     SCLogDebug("callback invoked by %u", s->id);
     AppLayerHtpEnableRequestBodyCallback();
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
 }
 
 /**
diff --git a/src/detect-http-cookie.c b/src/detect-http-cookie.c
index 1a24fffad4..90e02af79c 100644
--- a/src/detect-http-cookie.c
+++ b/src/detect-http-cookie.c
@@ -64,7 +64,6 @@
 static int DetectHttpCookieSetup (DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpCookieRegisterTests(void);
 static void DetectHttpCookieFree(void *);
-static void DetectHttpCookieSetupCallback(Signature *s);
 static int g_http_cookie_buffer_id = 0;
 
 /**
@@ -97,9 +96,6 @@ void DetectHttpCookieRegister(void)
     DetectBufferTypeSetDescriptionByName("http_cookie",
             "http cookie header");
 
-    DetectBufferTypeRegisterSetupCallback("http_cookie",
-            DetectHttpCookieSetupCallback);
-
     g_http_cookie_buffer_id = DetectBufferTypeGetByName("http_cookie");
 }
 
@@ -137,13 +133,6 @@ static int DetectHttpCookieSetup(DetectEngineCtx *de_ctx, Signature *s, const ch
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpCookieSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
-
 /******************************** UNITESTS **********************************/
 
 #ifdef UNITTESTS
diff --git a/src/detect-http-header-names.c b/src/detect-http-header-names.c
index 5410f15d89..0ac8f7aacc 100644
--- a/src/detect-http-header-names.c
+++ b/src/detect-http-header-names.c
@@ -365,12 +365,6 @@ static int DetectHttpHeaderNamesSetup(DetectEngineCtx *de_ctx, Signature *s, con
     return 0;
 }
 
-static void DetectHttpHeaderNamesSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief Registers the keyword handlers for the "http_header" keyword.
  */
@@ -398,9 +392,6 @@ void DetectHttpHeaderNamesRegister(void)
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME,
             BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectHttpHeaderNamesSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 
     g_keyword_thread_id = DetectRegisterThreadCtxGlobalFuncs(KEYWORD_NAME,
diff --git a/src/detect-http-header.c b/src/detect-http-header.c
index 522cc13a1a..9d56af61dd 100644
--- a/src/detect-http-header.c
+++ b/src/detect-http-header.c
@@ -65,7 +65,6 @@
 
 static int DetectHttpHeaderSetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpHeaderRegisterTests(void);
-static void DetectHttpHeaderSetupCallback(Signature *);
 static int g_http_header_buffer_id = 0;
 static int g_keyword_thread_id = 0;
 
@@ -367,12 +366,6 @@ static int DetectHttpHeaderSetup(DetectEngineCtx *de_ctx, Signature *s, const ch
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpHeaderSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief Registers the keyword handlers for the "http_header" keyword.
  */
@@ -401,9 +394,6 @@ void DetectHttpHeaderRegister(void)
     DetectBufferTypeSetDescriptionByName("http_header",
             "http headers");
 
-    DetectBufferTypeRegisterSetupCallback("http_header",
-            DetectHttpHeaderSetupCallback);
-
     g_http_header_buffer_id = DetectBufferTypeGetByName("http_header");
 
     g_keyword_thread_id = DetectRegisterThreadCtxGlobalFuncs("http_header",
diff --git a/src/detect-http-headers-stub.h b/src/detect-http-headers-stub.h
index 7144fa6a68..98f26a5cbd 100644
--- a/src/detect-http-headers-stub.h
+++ b/src/detect-http-headers-stub.h
@@ -322,12 +322,6 @@ static int DetectHttpHeadersSetup(DetectEngineCtx *de_ctx, Signature *s, const c
     return 0;
 }
 
-static void DetectHttpHeadersSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 static void DetectHttpHeadersRegisterStub(void)
 {
     sigmatch_table[KEYWORD_ID].name = KEYWORD_NAME;
@@ -356,8 +350,5 @@ static void DetectHttpHeadersRegisterStub(void)
 
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME, BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectHttpHeadersSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 }
diff --git a/src/detect-http-hh.c b/src/detect-http-hh.c
index ad92c06427..af0e2b4c6c 100644
--- a/src/detect-http-hh.c
+++ b/src/detect-http-hh.c
@@ -62,7 +62,6 @@
 static int DetectHttpHHSetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpHHRegisterTests(void);
 static void DetectHttpHHFree(void *);
-static void DetectHttpHostSetupCallback(Signature *s);
 static _Bool DetectHttpHostValidateCallback(const Signature *s, const char **sigerror);
 static int g_http_host_buffer_id = 0;
 
@@ -90,9 +89,6 @@ void DetectHttpHHRegister(void)
     DetectBufferTypeSetDescriptionByName("http_host",
             "http host header");
 
-    DetectBufferTypeRegisterSetupCallback("http_host",
-            DetectHttpHostSetupCallback);
-
     DetectBufferTypeRegisterValidateCallback("http_host",
             DetectHttpHostValidateCallback);
 
@@ -120,12 +116,6 @@ static int DetectHttpHHSetup(DetectEngineCtx *de_ctx, Signature *s, const char *
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpHostSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 static _Bool DetectHttpHostValidateCallback(const Signature *s, const char **sigerror)
 {
     const SigMatch *sm = s->init_data->smlists[g_http_host_buffer_id];
diff --git a/src/detect-http-hrh.c b/src/detect-http-hrh.c
index 0afd1d6ed9..1c5e439438 100644
--- a/src/detect-http-hrh.c
+++ b/src/detect-http-hrh.c
@@ -62,7 +62,6 @@
 static int DetectHttpHRHSetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpHRHRegisterTests(void);
 static void DetectHttpHRHFree(void *);
-static void DetectHttpHostRawSetupCallback(Signature *);
 static int g_http_raw_host_buffer_id = 0;
 
 /**
@@ -89,9 +88,6 @@ void DetectHttpHRHRegister(void)
     DetectBufferTypeSetDescriptionByName("http_raw_host",
             "http raw host header");
 
-    DetectBufferTypeRegisterSetupCallback("http_raw_host",
-            DetectHttpHostRawSetupCallback);
-
     g_http_raw_host_buffer_id = DetectBufferTypeGetByName("http_raw_host");
 }
 
@@ -116,12 +112,6 @@ int DetectHttpHRHSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg)
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpHostRawSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief The function to free the http_raw_host data.
  *
diff --git a/src/detect-http-method.c b/src/detect-http-method.c
index 72165c0ec1..c36b2b98a4 100644
--- a/src/detect-http-method.c
+++ b/src/detect-http-method.c
@@ -64,7 +64,6 @@ static int g_http_method_buffer_id = 0;
 static int DetectHttpMethodSetup(DetectEngineCtx *, Signature *, const char *);
 void DetectHttpMethodRegisterTests(void);
 void DetectHttpMethodFree(void *);
-static void DetectHttpMethodSetupCallback(Signature *s);
 static _Bool DetectHttpMethodValidateCallback(const Signature *s, const char **sigerror);
 
 /**
@@ -91,8 +90,6 @@ void DetectHttpMethodRegister(void)
     DetectBufferTypeSetDescriptionByName("http_method",
             "http request method");
 
-    DetectBufferTypeRegisterSetupCallback("http_method",
-            DetectHttpMethodSetupCallback);
     DetectBufferTypeRegisterValidateCallback("http_method",
             DetectHttpMethodValidateCallback);
 
@@ -134,12 +131,6 @@ void DetectHttpMethodFree(void *ptr)
     SCFree(data);
 }
 
-static void DetectHttpMethodSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  *  \retval 1 valid
  *  \retval 0 invalid
diff --git a/src/detect-http-protocol.c b/src/detect-http-protocol.c
index 0ca005a874..79f4fb0788 100644
--- a/src/detect-http-protocol.c
+++ b/src/detect-http-protocol.c
@@ -200,12 +200,6 @@ static int DetectHttpProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const
     return 0;
 }
 
-static void DetectHttpProtocolSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief Registers the keyword handlers for the "http_header" keyword.
  */
@@ -233,8 +227,5 @@ void DetectHttpProtocolRegister(void)
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME,
             BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectHttpProtocolSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 }
diff --git a/src/detect-http-raw-header.c b/src/detect-http-raw-header.c
index 8c99d0f944..388a26a7ec 100644
--- a/src/detect-http-raw-header.c
+++ b/src/detect-http-raw-header.c
@@ -64,7 +64,6 @@ static int DetectHttpRawHeaderSetup(DetectEngineCtx *, Signature *, const char *
 static void DetectHttpRawHeaderRegisterTests(void);
 static void DetectHttpRawHeaderFree(void *);
 static _Bool DetectHttpRawHeaderValidateCallback(const Signature *s, const char **sigerror);
-static void DetectHttpRawHeaderSetupCallback(Signature *s);
 static int g_http_raw_header_buffer_id = 0;
 
 /**
@@ -97,8 +96,6 @@ void DetectHttpRawHeaderRegister(void)
 
     DetectBufferTypeRegisterValidateCallback("http_raw_header",
             DetectHttpRawHeaderValidateCallback);
-    DetectBufferTypeRegisterSetupCallback("http_raw_header",
-            DetectHttpRawHeaderSetupCallback);
 
     g_http_raw_header_buffer_id = DetectBufferTypeGetByName("http_raw_header");
 }
@@ -157,12 +154,6 @@ static _Bool DetectHttpRawHeaderValidateCallback(const Signature *s, const char
     return TRUE;
 }
 
-static void DetectHttpRawHeaderSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /************************************Unittests*********************************/
 
 #ifdef UNITTESTS
diff --git a/src/detect-http-raw-uri.c b/src/detect-http-raw-uri.c
index 369fadd7cf..7f9bc4a741 100644
--- a/src/detect-http-raw-uri.c
+++ b/src/detect-http-raw-uri.c
@@ -121,7 +121,6 @@ static bool DetectHttpRawUriValidateCallback(const Signature *s, const char **si
 static void DetectHttpRawUriSetupCallback(Signature *s)
 {
     SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
     DetectUrilenApplyToContent(s, g_http_raw_uri_buffer_id);
 }
 
diff --git a/src/detect-http-request-line.c b/src/detect-http-request-line.c
index 0c194e1254..047beee252 100644
--- a/src/detect-http-request-line.c
+++ b/src/detect-http-request-line.c
@@ -67,7 +67,6 @@ static int DetectEngineInspectHttpRequestLine(ThreadVars *tv,
         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
         const Signature *s, const SigMatchData *smd,
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
-static void DetectHttpRequestLineSetupCallback(Signature *s);
 static int g_http_request_line_buffer_id = 0;
 
 /**
@@ -94,9 +93,6 @@ void DetectHttpRequestLineRegister(void)
     DetectBufferTypeSetDescriptionByName("http_request_line",
             "http request line");
 
-    DetectBufferTypeRegisterSetupCallback("http_request_line",
-            DetectHttpRequestLineSetupCallback);
-
     g_http_request_line_buffer_id = DetectBufferTypeGetByName("http_request_line");
 }
 
@@ -120,12 +116,6 @@ static int DetectHttpRequestLineSetup(DetectEngineCtx *de_ctx, Signature *s, con
     return 0;
 }
 
-static void DetectHttpRequestLineSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /** \brief HTTP request line Mpm prefilter callback
  *
  *  \param det_ctx detection engine thread ctx
diff --git a/src/detect-http-response-line.c b/src/detect-http-response-line.c
index 1f1e95606e..e375984fe3 100644
--- a/src/detect-http-response-line.c
+++ b/src/detect-http-response-line.c
@@ -67,7 +67,6 @@ static int DetectEngineInspectHttpResponseLine(ThreadVars *tv,
         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
         const Signature *s, const SigMatchData *smd,
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
-static void DetectHttpResponseLineSetupCallback(Signature *s);
 static int g_http_response_line_id = 0;
 
 /**
@@ -94,9 +93,6 @@ void DetectHttpResponseLineRegister(void)
     DetectBufferTypeSetDescriptionByName("http_response_line",
             "http response line");
 
-    DetectBufferTypeRegisterSetupCallback("http_response_line",
-            DetectHttpResponseLineSetupCallback);
-
     g_http_response_line_id = DetectBufferTypeGetByName("http_response_line");
 }
 
@@ -120,12 +116,6 @@ static int DetectHttpResponseLineSetup(DetectEngineCtx *de_ctx, Signature *s, co
     return 0;
 }
 
-static void DetectHttpResponseLineSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /** \brief HTTP response line Mpm prefilter callback
  *
  *  \param det_ctx detection engine thread ctx
diff --git a/src/detect-http-start.c b/src/detect-http-start.c
index 914ecca4cd..bc0d32f902 100644
--- a/src/detect-http-start.c
+++ b/src/detect-http-start.c
@@ -289,12 +289,6 @@ static int DetectHttpStartSetup(DetectEngineCtx *de_ctx, Signature *s, const cha
     return 0;
 }
 
-static void DetectHttpStartSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief Registers the keyword handlers for the "http_header" keyword.
  */
@@ -322,9 +316,6 @@ void DetectHttpStartRegister(void)
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME,
             BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectHttpStartSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 
     g_keyword_thread_id = DetectRegisterThreadCtxGlobalFuncs(KEYWORD_NAME,
diff --git a/src/detect-http-stat-code.c b/src/detect-http-stat-code.c
index 9fc9e40660..1a160316b1 100644
--- a/src/detect-http-stat-code.c
+++ b/src/detect-http-stat-code.c
@@ -65,7 +65,6 @@
 
 static int DetectHttpStatCodeSetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpStatCodeRegisterTests(void);
-static void DetectHttpStatCodeSetupCallback(Signature *);
 static int g_http_stat_code_buffer_id = 0;
 
 /**
@@ -93,9 +92,6 @@ void DetectHttpStatCodeRegister (void)
     DetectBufferTypeSetDescriptionByName("http_stat_code",
             "http response status code");
 
-    DetectBufferTypeRegisterSetupCallback("http_stat_code",
-            DetectHttpStatCodeSetupCallback);
-
     g_http_stat_code_buffer_id = DetectBufferTypeGetByName("http_stat_code");
 }
 
@@ -118,12 +114,6 @@ static int DetectHttpStatCodeSetup(DetectEngineCtx *de_ctx, Signature *s, const
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpStatCodeSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 #ifdef UNITTESTS
 
 /**
diff --git a/src/detect-http-stat-msg.c b/src/detect-http-stat-msg.c
index 49a793cad2..11f045cb5c 100644
--- a/src/detect-http-stat-msg.c
+++ b/src/detect-http-stat-msg.c
@@ -65,7 +65,6 @@
 
 static int DetectHttpStatMsgSetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpStatMsgRegisterTests(void);
-static void DetectHttpStatMsgSetupCallback(Signature *s);
 static int g_http_stat_msg_buffer_id = 0;
 
 /**
@@ -93,9 +92,6 @@ void DetectHttpStatMsgRegister (void)
     DetectBufferTypeSetDescriptionByName("http_stat_msg",
             "http response status message");
 
-    DetectBufferTypeRegisterSetupCallback("http_stat_msg",
-            DetectHttpStatMsgSetupCallback);
-
     g_http_stat_msg_buffer_id = DetectBufferTypeGetByName("http_stat_msg");
 }
 
@@ -118,12 +114,6 @@ static int DetectHttpStatMsgSetup(DetectEngineCtx *de_ctx, Signature *s, const c
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpStatMsgSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 #ifdef UNITTESTS
 
 /**
diff --git a/src/detect-http-ua.c b/src/detect-http-ua.c
index f2ca11ceb7..2e72869bdb 100644
--- a/src/detect-http-ua.c
+++ b/src/detect-http-ua.c
@@ -62,7 +62,6 @@
 static int DetectHttpUASetup(DetectEngineCtx *, Signature *, const char *);
 static void DetectHttpUARegisterTests(void);
 static void DetectHttpUAFree(void *);
-static void DetectHttpUASetupCallback(Signature *);
 static int g_http_ua_buffer_id = 0;
 
 /**
@@ -90,9 +89,6 @@ void DetectHttpUARegister(void)
     DetectBufferTypeSetDescriptionByName("http_user_agent",
             "http user agent");
 
-    DetectBufferTypeRegisterSetupCallback("http_user_agent",
-            DetectHttpUASetupCallback);
-
     g_http_ua_buffer_id = DetectBufferTypeGetByName("http_user_agent");
 }
 
@@ -117,12 +113,6 @@ int DetectHttpUASetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg)
                                                   ALPROTO_HTTP);
 }
 
-static void DetectHttpUASetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
-}
-
 /**
  * \brief The function to free the http_user_agent data.
  *
diff --git a/src/detect-http-uri.c b/src/detect-http-uri.c
index 6eaebbb358..2fe21da726 100644
--- a/src/detect-http-uri.c
+++ b/src/detect-http-uri.c
@@ -125,7 +125,6 @@ static bool DetectHttpUriValidateCallback(const Signature *s, const char **siger
 static void DetectHttpUriSetupCallback(Signature *s)
 {
     SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_HTTP_STATE;
     DetectUrilenApplyToContent(s, g_http_uri_buffer_id);
 }
 
diff --git a/src/detect-ssh-proto.c b/src/detect-ssh-proto.c
index 75688a6b3a..4039830452 100644
--- a/src/detect-ssh-proto.c
+++ b/src/detect-ssh-proto.c
@@ -176,12 +176,6 @@ static int DetectSshProtocolSetup(DetectEngineCtx *de_ctx, Signature *s, const c
     return 0;
 }
 
-static void DetectSshProtocolSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_SSH_STATE;
-}
-
 void DetectSshProtocolRegister(void)
 {
     sigmatch_table[DETECT_AL_SSH_PROTOCOL].name = KEYWORD_NAME;
@@ -205,8 +199,5 @@ void DetectSshProtocolRegister(void)
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME,
             BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectSshProtocolSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 }
diff --git a/src/detect-ssh-software.c b/src/detect-ssh-software.c
index abeccc06e6..2b8dc1114d 100644
--- a/src/detect-ssh-software.c
+++ b/src/detect-ssh-software.c
@@ -176,12 +176,6 @@ static int DetectSshSoftwareSetup(DetectEngineCtx *de_ctx, Signature *s, const c
     return 0;
 }
 
-static void DetectSshSoftwareSetupCallback(Signature *s)
-{
-    SCLogDebug("callback invoked by %u", s->id);
-    s->mask |= SIG_MASK_REQUIRE_SSH_STATE;
-}
-
 void DetectSshSoftwareRegister(void)
 {
     sigmatch_table[DETECT_AL_SSH_SOFTWARE].name = KEYWORD_NAME;
@@ -205,8 +199,5 @@ void DetectSshSoftwareRegister(void)
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME,
             BUFFER_DESC);
 
-    DetectBufferTypeRegisterSetupCallback(BUFFER_NAME,
-            DetectSshSoftwareSetupCallback);
-
     g_buffer_id = DetectBufferTypeGetByName(BUFFER_NAME);
 }
diff --git a/src/detect.c b/src/detect.c
index 7f038975bf..50758ac355 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -48,6 +48,7 @@
 #include "detect-engine-prefilter.h"
 #include "detect-engine-state.h"
 #include "detect-engine-analyzer.h"
+
 #include "detect-engine-filedata.h"
 
 
@@ -55,15 +56,6 @@
 #include "detect-engine-event.h"
 #include "detect-engine-hcbd.h"
 #include "detect-engine-hsbd.h"
-#include "detect-engine-hrhd.h"
-#include "detect-engine-hmd.h"
-#include "detect-engine-hcd.h"
-#include "detect-engine-hrud.h"
-#include "detect-engine-hsmd.h"
-#include "detect-engine-hscd.h"
-#include "detect-engine-hua.h"
-#include "detect-engine-hhhd.h"
-#include "detect-engine-hrhhd.h"
 
 #include "detect-filestore.h"
 #include "detect-flowvar.h"
@@ -72,15 +64,95 @@
 #include "util-validate.h"
 #include "util-detect.h"
 
-static void SigMatchSignaturesCleanup(DetectEngineThreadCtx *det_ctx,
-    Packet *p, Flow *pflow, const bool use_flow_sgh);
+typedef struct DetectRunScratchpad {
+    const AppProto alproto;
+    const uint8_t flow_flags; /* flow/state flags: STREAM_* */
+    const bool app_decoder_events;
+    const SigGroupHead *sgh;
+    SignatureMask pkt_mask;
+} DetectRunScratchpad;
+
+/* prototypes */
+static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow);
+static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p);
+static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx,
+        Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch);
+static inline void DetectRunPrefilterPkt(ThreadVars *tv,
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
+        DetectRunScratchpad *scratch);
+static inline void DetectRulePacketRules(ThreadVars * const tv,
+        DetectEngineCtx * const de_ctx, DetectEngineThreadCtx * const det_ctx,
+        Packet * const p, Flow * const pflow, const DetectRunScratchpad *scratch);
+static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Packet *p,
+        Flow *f, DetectRunScratchpad *scratch);
+static inline void DetectRunPostRules(ThreadVars *tv, DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Packet * const p, Flow * const pflow,
+        DetectRunScratchpad *scratch);
+static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
+        Packet *p, Flow * const pflow);
 
-int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
-                                   DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
-                                   const Signature *s)
+/** \internal
+ */
+static void DetectRun(ThreadVars *th_v,
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        Packet *p)
+{
+    SCEnter();
+    SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
+
+    /* bail early if packet should not be inspected */
+    if (p->flags & PKT_NOPACKET_INSPECTION) {
+        /* nothing to do */
+        SCReturn;
+    }
+
+    /* Load the Packet's flow early, even though it might not be needed.
+     * Mark as a constant pointer, although the flow itself can change. */
+    Flow * const pflow = p->flow;
+
+    DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow);
+
+    /* run the IPonly engine */
+    DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p);
+
+    /* get our rule group */
+    DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch);
+    /* if we didn't get a sig group head, we
+     * have nothing to do.... */
+    if (scratch.sgh == NULL) {
+        SCLogDebug("no sgh for this packet, nothing to match against");
+        goto end;
+    }
+
+    /* run the prefilters for packets */
+    DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch);
+
+    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES);
+    /* inspect the rules against the packet */
+    DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
+    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_RULES);
+
+    /* run tx/state inspection */
+    if (pflow && pflow->alstate) {
+        DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch);
+    }
+
+end:
+    DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch);
+
+    DetectRunCleanup(det_ctx, p, pflow);
+    SCReturn;
+}
+
+static void DetectRunPostMatch(ThreadVars *tv,
+                               DetectEngineThreadCtx *det_ctx, Packet *p,
+                               const Signature *s)
 {
     /* run the packet match functions */
-    SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_POSTMATCH];
+    const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_POSTMATCH];
     if (smd != NULL) {
         KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_POSTMATCH);
 
@@ -101,7 +173,7 @@ int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
     if (s->flags & SIG_FLAG_FILESTORE)
         DetectFilestorePostMatch(tv, det_ctx, p, s);
 
-    return 1;
+    return;
 }
 
 /**
@@ -289,14 +361,15 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
 }
 
 static inline void
-DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask)
+DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask, uint8_t alproto)
 {
     uint32_t x = 0;
     for (x = 0; x < det_ctx->non_pf_store_cnt; x++) {
         /* only if the mask matches this rule can possibly match,
          * so build the non_mpm array only for match candidates */
-        SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
-        if ((rule_mask & mask) == rule_mask) {
+        const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
+        const uint8_t rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
+        if ((rule_mask & mask) == rule_mask && (rule_alproto == 0 || rule_alproto == alproto)) { // TODO dce?
             det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
         }
     }
@@ -304,21 +377,22 @@ DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMa
 
 /** \internal
  *  \brief select non-mpm list
- *  Based on the packet properties, select the non-mpm list to use */
+ *  Based on the packet properties, select the non-mpm list to use
+ *  \todo move non_pf_store* into scratchpad */
 static inline void
-DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx)
+DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
 {
     if ((p->proto == IPPROTO_TCP) && (p->tcph != NULL) && (p->tcph->th_flags & TH_SYN)) {
-        det_ctx->non_pf_store_ptr = det_ctx->sgh->non_pf_syn_store_array;
-        det_ctx->non_pf_store_cnt = det_ctx->sgh->non_pf_syn_store_cnt;
+        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
+        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
     } else {
-        det_ctx->non_pf_store_ptr = det_ctx->sgh->non_pf_other_store_array;
-        det_ctx->non_pf_store_cnt = det_ctx->sgh->non_pf_other_store_cnt;
+        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
+        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
     }
     SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
             det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
-            det_ctx->sgh->non_pf_syn_store_array, det_ctx->sgh->non_pf_syn_store_cnt,
-            det_ctx->sgh->non_pf_other_store_array, det_ctx->sgh->non_pf_other_store_cnt);
+            scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
+            scratch->sgh->non_pf_other_store_array, scratch->sgh->non_pf_other_store_cnt);
 }
 
 /** \internal
@@ -375,7 +449,7 @@ DetectPostInspectFileFlagsUpdate(Flow *pflow, const SigGroupHead *sgh, uint8_t d
 }
 
 static inline void
-DetectPostInspectFirstSGH(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
+DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
 {
     if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) {
         /* first time we see this toserver sgh, store it */
@@ -410,6 +484,104 @@ DetectPostInspectFirstSGH(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
     }
 }
 
+static inline void DetectRunGetRuleGroup(
+    const DetectEngineCtx *de_ctx,
+    Packet * const p, Flow * const pflow,
+    DetectRunScratchpad *scratch)
+{
+    const SigGroupHead *sgh = NULL;
+
+    if (pflow) {
+        bool use_flow_sgh = false;
+        /* Get the stored sgh from the flow (if any). Make sure we're not using
+         * the sgh for icmp error packets part of the same stream. */
+        if (IP_GET_IPPROTO(p) == pflow->proto) { /* filter out icmp */
+            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
+            if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
+                sgh = pflow->sgh_toserver;
+                SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh);
+                use_flow_sgh = true;
+            } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
+                sgh = pflow->sgh_toclient;
+                SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh);
+                use_flow_sgh = true;
+            }
+            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
+        }
+
+        if (!(use_flow_sgh)) {
+            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
+            sgh = SigMatchSignaturesGetSgh(de_ctx, p);
+            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
+
+            /* HACK: prevent the wrong sgh (or NULL) from being stored in the
+             * flow's sgh pointers */
+            if (PKT_IS_ICMPV4(p) && ICMPV4_DEST_UNREACH_IS_VALID(p)) {
+                ; /* no-op */
+            } else {
+                /* store the found sgh (or NULL) in the flow to save us
+                 * from looking it up again for the next packet.
+                 * Also run other tasks */
+                DetectRunPostGetFirstRuleGroup(p, pflow, sgh);
+            }
+        }
+    } else { /* p->flags & PKT_HAS_FLOW */
+        /* no flow */
+
+        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
+        sgh = SigMatchSignaturesGetSgh(de_ctx, p);
+        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
+    }
+
+    scratch->sgh = sgh;
+}
+
+static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx,
+        Flow * const pflow, Packet * const p)
+{
+    if (pflow) {
+        /* set the iponly stuff */
+        if (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)
+            p->flowflags |= FLOW_PKT_TOCLIENT_IPONLY_SET;
+        if (pflow->flags & FLOW_TOSERVER_IPONLY_SET)
+            p->flowflags |= FLOW_PKT_TOSERVER_IPONLY_SET;
+
+        if (((p->flowflags & FLOW_PKT_TOSERVER) && !(p->flowflags & FLOW_PKT_TOSERVER_IPONLY_SET)) ||
+            ((p->flowflags & FLOW_PKT_TOCLIENT) && !(p->flowflags & FLOW_PKT_TOCLIENT_IPONLY_SET)))
+        {
+            SCLogDebug("testing against \"ip-only\" signatures");
+
+            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY);
+            IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, &det_ctx->io_ctx, p);
+            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY);
+
+            /* save in the flow that we scanned this direction... */
+            FlowSetIPOnlyFlag(pflow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
+
+        } else if (((p->flowflags & FLOW_PKT_TOSERVER) &&
+                   (pflow->flags & FLOW_TOSERVER_IPONLY_SET)) ||
+                   ((p->flowflags & FLOW_PKT_TOCLIENT) &&
+                   (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)))
+        {
+            /* If we have a drop from IP only module,
+             * we will drop the rest of the flow packets
+             * This will apply only to inline/IPS */
+            if (pflow->flags & FLOW_ACTION_DROP) {
+                PACKET_DROP(p);
+            }
+        }
+    } else { /* p->flags & PKT_HAS_FLOW */
+        /* no flow */
+
+        /* Even without flow we should match the packet src/dst */
+        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY);
+        IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx,
+                          &det_ctx->io_ctx, p);
+        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY);
+    }
+}
+
 /* returns 0 if no match, 1 if match */
 static inline int DetectRunInspectRuleHeader(
     const Packet *p,
@@ -529,251 +701,79 @@ static inline int DetectRunInspectRulePacketMatches(
     return 1;
 }
 
-/**
- *  \brief Signature match function
+/** \internal
+ *  \brief run packet/stream prefilter engines
  */
-void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
+static inline void DetectRunPrefilterPkt(
+    ThreadVars *tv,
+    DetectEngineCtx *de_ctx,
+    DetectEngineThreadCtx *det_ctx,
+    Packet *p,
+    DetectRunScratchpad *scratch
+)
 {
-    bool use_flow_sgh = false;
-    uint8_t alert_flags = 0;
-    AppProto alproto = ALPROTO_UNKNOWN;
-    uint8_t flow_flags = 0; /* flow/state flags */
-    const Signature *s = NULL;
-    const Signature *next_s = NULL;
-    bool app_decoder_events = false;
-    bool has_state = false;     /* do we have an alstate to work with? */
-
-    SCEnter();
-
-    SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt);
-#ifdef UNITTESTS
-    p->alerts.cnt = 0;
-#endif
-    det_ctx->ticker++;
-    det_ctx->filestore_cnt = 0;
-    det_ctx->base64_decoded_len = 0;
-    det_ctx->raw_stream_progress = 0;
-
-#ifdef DEBUG
-    if (p->flags & PKT_STREAM_ADD) {
-        det_ctx->pkt_stream_add_cnt++;
-    }
-#endif
-
-    /* No need to perform any detection on this packet, if the the given flag is set.*/
-    if (p->flags & PKT_NOPACKET_INSPECTION) {
-        SCReturn;
-    }
-
-    /* Load the Packet's flow early, even though it might not be needed.
-     * Mark as a constant pointer, although the flow can change.
-     */
-    Flow * const pflow = p->flow;
-
-    /* grab the protocol state we will detect on */
-    if (p->flags & PKT_HAS_FLOW) {
-        if (p->flowflags & FLOW_PKT_TOSERVER) {
-            flow_flags = STREAM_TOSERVER;
-            SCLogDebug("flag STREAM_TOSERVER set");
-        } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
-            flow_flags = STREAM_TOCLIENT;
-            SCLogDebug("flag STREAM_TOCLIENT set");
-        }
-        SCLogDebug("p->flowflags 0x%02x", p->flowflags);
-
-        if (p->flags & PKT_STREAM_EOF) {
-            flow_flags |= STREAM_EOF;
-            SCLogDebug("STREAM_EOF set");
-        }
-
-        /* store tenant_id in the flow so that we can use it
-         * for creating pseudo packets */
-        if (p->tenant_id > 0 && pflow->tenant_id == 0) {
-            pflow->tenant_id = p->tenant_id;
-        }
-
-        /* live ruleswap check for flow updates */
-        if (pflow->de_ctx_version == 0) {
-            /* first time this flow is inspected, set id */
-            pflow->de_ctx_version = de_ctx->version;
-        } else if (pflow->de_ctx_version != de_ctx->version) {
-            /* first time we inspect flow with this de_ctx, reset */
-            pflow->flags &= ~FLOW_SGH_TOSERVER;
-            pflow->flags &= ~FLOW_SGH_TOCLIENT;
-            pflow->sgh_toserver = NULL;
-            pflow->sgh_toclient = NULL;
-
-            pflow->de_ctx_version = de_ctx->version;
-            GenericVarFree(pflow->flowvar);
-            pflow->flowvar = NULL;
-
-            DetectEngineStateResetTxs(pflow);
-        }
-
-        /* set the iponly stuff */
-        if (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)
-            p->flowflags |= FLOW_PKT_TOCLIENT_IPONLY_SET;
-        if (pflow->flags & FLOW_TOSERVER_IPONLY_SET)
-            p->flowflags |= FLOW_PKT_TOSERVER_IPONLY_SET;
-
-        /* Get the stored sgh from the flow (if any). Make sure we're not using
-         * the sgh for icmp error packets part of the same stream. */
-        if (IP_GET_IPPROTO(p) == pflow->proto) { /* filter out icmp */
-            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
-            if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) {
-                det_ctx->sgh = pflow->sgh_toserver;
-                SCLogDebug("det_ctx->sgh = pflow->sgh_toserver; => %p", det_ctx->sgh);
-                use_flow_sgh = true;
-            } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) {
-                det_ctx->sgh = pflow->sgh_toclient;
-                SCLogDebug("det_ctx->sgh = pflow->sgh_toclient; => %p", det_ctx->sgh);
-                use_flow_sgh = true;
-            }
-            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
-        }
-
-        /* Retrieve the app layer state and protocol and the tcp reassembled
-         * stream chunks. */
-        if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
-                (p->proto == IPPROTO_UDP) ||
-                (p->proto == IPPROTO_SCTP && (p->flowflags & FLOW_PKT_ESTABLISHED)))
-        {
-            /* update flow flags with knowledge on disruptions */
-            flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
-            has_state = (FlowGetAppState(pflow) != NULL);
-            alproto = FlowGetAppProtocol(pflow);
-            if (p->proto == IPPROTO_TCP && pflow->protoctx &&
-                    StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
-                p->flags |= PKT_DETECT_HAS_STREAMDATA;
-            }
-            SCLogDebug("alstate %s, alproto %u", has_state ? "true" : "false", alproto);
-        } else {
-            SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
-        }
-
-        app_decoder_events = AppLayerParserHasDecoderEvents(pflow,
-                pflow->alstate,
-                pflow->alparser,
-                flow_flags);
-
-        if (((p->flowflags & FLOW_PKT_TOSERVER) && !(p->flowflags & FLOW_PKT_TOSERVER_IPONLY_SET)) ||
-            ((p->flowflags & FLOW_PKT_TOCLIENT) && !(p->flowflags & FLOW_PKT_TOCLIENT_IPONLY_SET)))
-        {
-            SCLogDebug("testing against \"ip-only\" signatures");
-
-            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY);
-            IPOnlyMatchPacket(th_v, de_ctx, det_ctx, &de_ctx->io_ctx, &det_ctx->io_ctx, p);
-            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY);
-
-            /* save in the flow that we scanned this direction... */
-            FlowSetIPOnlyFlag(pflow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
-
-        } else if (((p->flowflags & FLOW_PKT_TOSERVER) &&
-                   (pflow->flags & FLOW_TOSERVER_IPONLY_SET)) ||
-                   ((p->flowflags & FLOW_PKT_TOCLIENT) &&
-                   (pflow->flags & FLOW_TOCLIENT_IPONLY_SET)))
-        {
-            /* If we have a drop from IP only module,
-             * we will drop the rest of the flow packets
-             * This will apply only to inline/IPS */
-            if (pflow->flags & FLOW_ACTION_DROP)
-            {
-                alert_flags = PACKET_ALERT_FLAG_DROP_FLOW;
-                PACKET_DROP(p);
-            }
-        }
-
-        if (!(use_flow_sgh)) {
-            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
-            det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, p);
-            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
-        }
-
-    } else { /* p->flags & PKT_HAS_FLOW */
-        /* no flow */
-
-        /* Even without flow we should match the packet src/dst */
-        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY);
-        IPOnlyMatchPacket(th_v, de_ctx, det_ctx, &de_ctx->io_ctx,
-                          &det_ctx->io_ctx, p);
-        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY);
-
-        PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH);
-        det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, p);
-        PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
-    }
-
-    /* if we didn't get a sig group head, we
-     * have nothing to do.... */
-    if (det_ctx->sgh == NULL) {
-        SCLogDebug("no sgh for this packet, nothing to match against");
-        goto end;
-    }
-
-    DetectPrefilterSetNonPrefilterList(p, det_ctx);
-
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_STATEFUL_CONT);
-    /* stateful app layer detection */
-    if ((p->flags & PKT_HAS_FLOW) && has_state) {
-        memset(det_ctx->de_state_sig_array, 0x00, det_ctx->de_state_sig_array_len);
-        int has_inspectable_state = DeStateFlowHasInspectableState(pflow, flow_flags);
-        if (has_inspectable_state == 1) {
-            /* initialize to 0(DE_STATE_MATCH_HAS_NEW_STATE) */
-            DeStateDetectContinueDetection(th_v, de_ctx, det_ctx, p, pflow,
-                                           flow_flags, alproto);
-        }
-    }
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_STATEFUL_CONT);
+    DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
 
     /* create our prefilter mask */
-    SignatureMask mask = 0;
-    PacketCreateMask(p, &mask, alproto, has_state, app_decoder_events);
+    PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
 
     /* build and prefilter non_pf list against the mask of the packet */
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
     det_ctx->non_pf_id_cnt = 0;
     if (likely(det_ctx->non_pf_store_cnt > 0)) {
-        DetectPrefilterBuildNonPrefilterList(det_ctx, mask);
+        DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
     }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST);
 
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER);
     /* run the prefilter engines */
-    Prefilter(det_ctx, det_ctx->sgh, p, flow_flags, has_state);
+    Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags);
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT2);
     DetectPrefilterMergeSort(de_ctx, det_ctx);
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT2);
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER);
 
 #ifdef PROFILING
-    if (th_v) {
-        StatsAddUI64(th_v, det_ctx->counter_mpm_list,
+    if (tv) {
+        StatsAddUI64(tv, det_ctx->counter_mpm_list,
                              (uint64_t)det_ctx->pmq.rule_id_array_cnt);
-        StatsAddUI64(th_v, det_ctx->counter_nonmpm_list,
+        StatsAddUI64(tv, det_ctx->counter_nonmpm_list,
                              (uint64_t)det_ctx->non_pf_store_cnt);
         /* non mpm sigs after mask prefilter */
-        StatsAddUI64(th_v, det_ctx->counter_fnonmpm_list,
+        StatsAddUI64(tv, det_ctx->counter_fnonmpm_list,
                              (uint64_t)det_ctx->non_pf_id_cnt);
     }
 #endif
+}
+
+static inline void DetectRulePacketRules(
+    ThreadVars * const tv,
+    DetectEngineCtx * const de_ctx,
+    DetectEngineThreadCtx * const det_ctx,
+    Packet * const p,
+    Flow * const pflow,
+    const DetectRunScratchpad *scratch
+)
+{
+    const Signature *s = NULL;
+    const Signature *next_s = NULL;
 
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES);
     /* inspect the sigs against the packet */
     /* Prefetch the next signature. */
     SigIntId match_cnt = det_ctx->match_array_cnt;
 #ifdef PROFILING
-    if (th_v) {
-        StatsAddUI64(th_v, det_ctx->counter_match_list,
+    if (tv) {
+        StatsAddUI64(tv, det_ctx->counter_match_list,
                              (uint64_t)match_cnt);
     }
 #endif
     Signature **match_array = det_ctx->match_array;
 
-    SGH_PROFILING_RECORD(det_ctx, det_ctx->sgh);
+    SGH_PROFILING_RECORD(det_ctx, scratch->sgh);
 #ifdef PROFILING
 #ifdef HAVE_LIBJANSSON
     if (match_cnt >= de_ctx->profile_match_logging_threshold)
-        RulesDumpMatchArray(det_ctx, p);
+        RulesDumpMatchArray(det_ctx, scratch->sgh, p);
 #endif
 #endif
 
@@ -784,6 +784,7 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
     }
     while (match_cnt--) {
         RULE_PROFILING_START(p);
+        uint8_t alert_flags = 0;
         bool state_alert = false;
 #ifdef PROFILING
         bool smatch = false; /* signature match */
@@ -799,30 +800,29 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
         SCLogDebug("inspecting signature id %"PRIu32"", s->id);
 
         if (sflags & SIG_FLAG_STATE_MATCH) {
-            if (det_ctx->de_state_sig_array[s->num] & DE_STATE_MATCH_NO_NEW_STATE)
-                goto next;
-        } else {
-            /* don't run mask check for stateful rules.
-             * There we depend on prefilter */
-            if ((s->mask & mask) != s->mask) {
-                SCLogDebug("mask mismatch %x & %x != %x", s->mask, mask, s->mask);
-                goto next;
-            }
+            goto next; // TODO skip and handle in DetectRunTx
+        }
 
-            if (unlikely(sflags & SIG_FLAG_DSIZE)) {
-                if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
-                    SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
-                            p->payload_len, s->dsize_low, s->dsize_high);
-                    goto next;
-                }
+        /* don't run mask check for stateful rules.
+         * There we depend on prefilter */
+        if ((s->mask & scratch->pkt_mask) != s->mask) {
+            SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
+            goto next;
+        }
+
+        if (unlikely(sflags & SIG_FLAG_DSIZE)) {
+            if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
+                SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
+                        p->payload_len, s->dsize_low, s->dsize_high);
+                goto next;
             }
         }
 
         /* if the sig has alproto and the session as well they should match */
         if (likely(sflags & SIG_FLAG_APPLAYER)) {
-            if (s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) {
+            if (s->alproto != ALPROTO_UNKNOWN && s->alproto != scratch->alproto) {
                 if (s->alproto == ALPROTO_DCERPC) {
-                    if (alproto != ALPROTO_SMB && alproto != ALPROTO_SMB2) {
+                    if (scratch->alproto != ALPROTO_SMB && scratch->alproto != ALPROTO_SMB2) {
                         SCLogDebug("DCERPC sig, alproto not SMB or SMB2");
                         goto next;
                     }
@@ -878,41 +878,13 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
             }
         }
 
-        if (DetectRunInspectRulePacketMatches(th_v, det_ctx, p, pflow, s) == 0)
+        if (DetectRunInspectRulePacketMatches(tv, det_ctx, p, pflow, s) == 0)
             goto next;
 
-        /* consider stateful sig matches */
-        if (sflags & SIG_FLAG_STATE_MATCH) {
-            if (has_state == false) {
-                SCLogDebug("state matches but no state, we can't match");
-                goto next;
-            }
-
-            SCLogDebug("stateful app layer match inspection starting");
-
-            /* if DeStateDetectStartDetection matches, it's a full
-             * signature match. It will then call PacketAlertAppend
-             * itself, so we can skip it below. This is done so it
-             * can store the tx_id with the alert */
-            PACKET_PROFILING_DETECT_START(p, PROF_DETECT_STATEFUL_START);
-            state_alert = DeStateDetectStartDetection(th_v, de_ctx, det_ctx, s,
-                                                      p, pflow, flow_flags, alproto);
-            PACKET_PROFILING_DETECT_END(p, PROF_DETECT_STATEFUL_START);
-            if (state_alert == false)
-                goto next;
-
-            /* match */
-            if (s->action & ACTION_DROP)
-                alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
-
-            alert_flags |= PACKET_ALERT_FLAG_STATE_MATCH;
-        }
-
 #ifdef PROFILING
         smatch = true;
 #endif
-
-        SigMatchSignaturesRunPostMatch(th_v, de_ctx, det_ctx, p, s);
+        DetectRunPostMatch(tv, det_ctx, p, s);
 
         if (!(sflags & SIG_FLAG_NOALERT)) {
             /* stateful sigs call PacketAlertAppend from DeStateDetectStartDetection */
@@ -930,13 +902,108 @@ next:
         det_ctx->flags = 0;
         continue;
     }
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_RULES);
+}
 
-end:
+static DetectRunScratchpad DetectRunSetup(
+    const DetectEngineCtx *de_ctx,
+    DetectEngineThreadCtx *det_ctx,
+    Packet * const p, Flow * const pflow)
+{
+    AppProto alproto = ALPROTO_UNKNOWN;
+    uint8_t flow_flags = 0; /* flow/state flags */
+    bool app_decoder_events = false;
+
+#ifdef UNITTESTS
+    p->alerts.cnt = 0;
+#endif
+    det_ctx->ticker++;
+    det_ctx->filestore_cnt = 0;
+    det_ctx->base64_decoded_len = 0;
+    det_ctx->raw_stream_progress = 0;
+
+#ifdef DEBUG
+    if (p->flags & PKT_STREAM_ADD) {
+        det_ctx->pkt_stream_add_cnt++;
+    }
+#endif
+
+    /* grab the protocol state we will detect on */
+    if (p->flags & PKT_HAS_FLOW) {
+        if (p->flowflags & FLOW_PKT_TOSERVER) {
+            flow_flags = STREAM_TOSERVER;
+            SCLogDebug("flag STREAM_TOSERVER set");
+        } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
+            flow_flags = STREAM_TOCLIENT;
+            SCLogDebug("flag STREAM_TOCLIENT set");
+        }
+        SCLogDebug("p->flowflags 0x%02x", p->flowflags);
+
+        if (p->flags & PKT_STREAM_EOF) {
+            flow_flags |= STREAM_EOF;
+            SCLogDebug("STREAM_EOF set");
+        }
+
+        /* store tenant_id in the flow so that we can use it
+         * for creating pseudo packets */
+        if (p->tenant_id > 0 && pflow->tenant_id == 0) {
+            pflow->tenant_id = p->tenant_id;
+        }
+
+        /* live ruleswap check for flow updates */
+        if (pflow->de_ctx_version == 0) {
+            /* first time this flow is inspected, set id */
+            pflow->de_ctx_version = de_ctx->version;
+        } else if (pflow->de_ctx_version != de_ctx->version) {
+            /* first time we inspect flow with this de_ctx, reset */
+            pflow->flags &= ~FLOW_SGH_TOSERVER;
+            pflow->flags &= ~FLOW_SGH_TOCLIENT;
+            pflow->sgh_toserver = NULL;
+            pflow->sgh_toclient = NULL;
+
+            pflow->de_ctx_version = de_ctx->version;
+            GenericVarFree(pflow->flowvar);
+            pflow->flowvar = NULL;
+
+            DetectEngineStateResetTxs(pflow);
+        }
+
+        /* Retrieve the app layer state and protocol and the tcp reassembled
+         * stream chunks. */
+        if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) ||
+                (p->proto == IPPROTO_UDP) ||
+                (p->proto == IPPROTO_SCTP && (p->flowflags & FLOW_PKT_ESTABLISHED)))
+        {
+            /* update flow flags with knowledge on disruptions */
+            flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
+            alproto = FlowGetAppProtocol(pflow);
+            if (p->proto == IPPROTO_TCP && pflow->protoctx &&
+                    StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
+                p->flags |= PKT_DETECT_HAS_STREAMDATA;
+            }
+            SCLogDebug("alproto %u", alproto);
+        } else {
+            SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
+        }
+
+        app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
+    }
+
+    DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 };
+    return pad;
+}
+
+static inline void DetectRunPostRules(
+    ThreadVars *tv,
+    DetectEngineCtx *de_ctx,
+    DetectEngineThreadCtx *det_ctx,
+    Packet * const p,
+    Flow * const pflow,
+    DetectRunScratchpad *scratch)
+{
     /* see if we need to increment the inspect_id and reset the de_state */
-    if (has_state && AppLayerParserProtocolSupportsTxs(p->proto, alproto)) {
+    if (pflow && pflow->alstate && AppLayerParserProtocolSupportsTxs(p->proto, scratch->alproto)) {
         PACKET_PROFILING_DETECT_START(p, PROF_DETECT_STATEFUL_UPDATE);
-        DeStateUpdateInspectTransactionId(pflow, flow_flags);
+        DeStateUpdateInspectTransactionId(pflow, scratch->flow_flags, (scratch->sgh == NULL));
         PACKET_PROFILING_DETECT_END(p, PROF_DETECT_STATEFUL_UPDATE);
     }
 
@@ -947,35 +1014,19 @@ end:
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_ALERT);
     PacketAlertFinalize(de_ctx, det_ctx, p);
     if (p->alerts.cnt > 0) {
-        StatsAddUI64(th_v, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
+        StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
     }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_ALERT);
-
-    SigMatchSignaturesCleanup(det_ctx, p, pflow, use_flow_sgh);
-    SCReturn;
 }
 
-/* TODO move use_flow_sgh into det_ctx */
-static void SigMatchSignaturesCleanup(DetectEngineThreadCtx *det_ctx,
-    Packet *p, Flow * const pflow, const bool use_flow_sgh)
+static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
+        Packet *p, Flow * const pflow)
 {
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_CLEANUP);
     /* cleanup pkt specific part of the patternmatcher */
     PacketPatternCleanup(det_ctx);
 
-    /* store the found sgh (or NULL) in the flow to save us from looking it
-     * up again for the next packet. Also return any stream chunk we processed
-     * to the pool. */
     if (pflow != NULL) {
-        /* HACK: prevent the wrong sgh (or NULL) from being stored in the
-         * flow's sgh pointers */
-        if (PKT_IS_ICMPV4(p) && ICMPV4_DEST_UNREACH_IS_VALID(p)) {
-            ; /* no-op */
-
-        } else if (!(use_flow_sgh)) {
-            DetectPostInspectFirstSGH(p, pflow, det_ctx->sgh);
-        }
-
         /* update inspected tracker for raw reassembly */
         if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL) {
             StreamReassembleRawUpdateProgress(pflow->protoctx, p,
@@ -990,6 +1041,498 @@ static void SigMatchSignaturesCleanup(DetectEngineThreadCtx *det_ctx,
     SCReturn;
 }
 
+void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size)
+{
+    DEBUG_VALIDATE_BUG_ON(det_ctx->tx_candidates);
+    det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx));
+    if (det_ctx->tx_candidates == NULL) {
+        FatalError(SC_ERR_MEM_ALLOC, "failed to allocate %"PRIu64" bytes",
+                (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
+    }
+    det_ctx->tx_candidates_size = size;
+    SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)",
+            size, (uint64_t)(size * sizeof(RuleMatchCandidateTx)));
+}
+
+void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx)
+{
+    SCFree(det_ctx->tx_candidates);
+    det_ctx->tx_candidates_size = 0;
+}
+
+/* if size >= cur_space */
+static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx,
+        const uint32_t need)
+{
+    if (det_ctx->tx_candidates_size >= need)
+        return 1;
+    return 0;
+}
+
+/* realloc */
+static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed)
+{
+    const uint32_t old_size = det_ctx->tx_candidates_size;
+    uint32_t new_size = needed;
+    void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx)));
+    if (ptmp == NULL) {
+        FatalError(SC_ERR_MEM_ALLOC, "failed to expand to %"PRIu64" bytes",
+                (uint64_t)(new_size * sizeof(RuleMatchCandidateTx)));
+        // TODO can this be handled more gracefully?
+    }
+    det_ctx->tx_candidates = ptmp;
+    det_ctx->tx_candidates_size = new_size;
+    SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)",
+            old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)),
+            (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size;
+    return 1;
+}
+
+
+/* TODO maybe let one with flags win if equal? */
+static int
+DetectRunTxSortHelper(const void *a, const void *b)
+{
+    const RuleMatchCandidateTx *s0 = a;
+    const RuleMatchCandidateTx *s1 = b;
+    if (s1->id == s0->id)
+        return 0;
+    else
+        return s0->id > s1->id ? -1 : 1;
+}
+
+#if 0
+#define TRACE_SID_TXS(sid,txs,...)          \
+    do {                                    \
+        char _trace_buf[2048];              \
+        snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__);  \
+        SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf);   \
+    } while(0)
+#else
+#define TRACE_SID_TXS(sid,txs,...)
+#endif
+
+/** \internal
+ *  \brief inspect a rule against a transaction
+ *
+ *  Inspect a rule. New detection or continued stateful
+ *  detection.
+ *
+ *  \param stored_flags pointer to stored flags or NULL.
+ *         If stored_flags is set it means we're continueing
+ *         inspection from an earlier run.
+ *
+ *  \retval bool true sig matched, false didn't match
+ */
+static bool DetectRunTxInspectRule(ThreadVars *tv,
+        DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx,
+        Packet *p,
+        Flow *f,
+        const uint8_t flow_flags,   // direction, EOF, etc
+        void *alstate,
+        DetectTransaction *tx,
+        const Signature *s,
+        uint32_t *stored_flags,
+        RuleMatchCandidateTx *can,
+        DetectRunScratchpad *scratch)
+{
+    const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
+    uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
+    int total_matches = 0;
+    int file_no_match = 0;
+    bool retval = false;
+    bool mpm_before_progress = false;   // is mpm engine before progress?
+    bool mpm_in_progress = false;       // is mpm engine in a buffer we will revisit?
+
+    TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver");
+
+    /* for a new inspection we inspect pkt header and packet matches */
+    if (likely(stored_flags == NULL)) {
+        TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
+        if (DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags) == 0) {
+            TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
+            return false;
+        }
+        if (DetectRunInspectRulePacketMatches(tv, det_ctx, p, f, s) == 0) {
+            TRACE_SID_TXS(s->id, tx, "DetectRunInspectRulePacketMatches no match");
+            return false;
+        }
+        /* stream mpm and negated mpm sigs can end up here with wrong proto */
+        if (!(f->alproto == s->alproto || s->alproto == ALPROTO_UNKNOWN)) {
+            TRACE_SID_TXS(s->id, tx, "alproto mismatch");
+            return false;
+        }
+    }
+
+    const DetectEngineAppInspectionEngine *engine = s->app_inspect;
+    while (engine != NULL) { // TODO could be do {} while as s->app_inspect cannot be null
+        TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags);
+        if (!(inspect_flags & BIT_U32(engine->id)) &&
+                direction == engine->dir)
+        {
+            /* engines are sorted per progress, except that the one with
+             * mpm/prefilter enabled is first */
+            if (tx->tx_progress < engine->progress) {
+                SCLogDebug("tx progress %d < engine progress %d",
+                        tx->tx_progress, engine->progress);
+                break;
+            }
+            if (engine->mpm) {
+                if (tx->tx_progress > engine->progress) {
+                    mpm_before_progress = true;
+                } else if (tx->tx_progress == engine->progress) {
+                    mpm_in_progress = true;
+                }
+            }
+
+            /* run callback: but bypass stream callback if we can */
+            int match;
+            if (unlikely(engine->stream && can->stream_stored)) {
+                match = can->stream_result;
+                TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
+            /* special case: file_data on 'alert tcp' will have engines
+             * in the list that are not for us. Bypass with assume match */
+            } else if (unlikely(engine->alproto != 0 && engine->alproto != f->alproto)) {
+                inspect_flags |= BIT_U32(engine->id);
+                engine = engine->next;
+                total_matches++;
+                continue;
+            } else {
+                KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
+                match = engine->Callback(tv, de_ctx, det_ctx,
+                        s, engine->smd, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
+                TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
+                if (engine->stream) {
+                    can->stream_stored = true;
+                    can->stream_result = match;
+                    TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match);
+                }
+            }
+            if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
+                inspect_flags |= BIT_U32(engine->id);
+                engine = engine->next;
+                total_matches++;
+                continue;
+            } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
+                /* if the file engine matched, but indicated more
+                 * files are still in progress, we don't set inspect
+                 * flags as these would end inspection for this tx */
+                engine = engine->next;
+                total_matches++;
+                continue;
+            } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
+                inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
+                inspect_flags |= BIT_U32(engine->id);
+            } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
+                inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
+                inspect_flags |= BIT_U32(engine->id);
+                file_no_match = 1;
+            }
+            /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */
+            if (engine->mpm && mpm_before_progress) {
+                inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
+                inspect_flags |= BIT_U32(engine->id);
+            }
+            break;
+        }
+        engine = engine->next;
+    }
+    TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
+            inspect_flags, total_matches, engine);
+
+    if (engine == NULL && total_matches) {
+        inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
+        TRACE_SID_TXS(s->id, tx, "MATCH");
+        retval = true;
+    }
+
+    if (stored_flags) {
+        *stored_flags = inspect_flags;
+        TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags);
+    } else {
+        // store... or? If tx is done we might not want to come back to this tx
+
+        // also... if mpmid tracking is enabled, we won't do a sig again for this tx...
+        TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
+        if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
+            if (file_no_match) {
+                DetectRunStoreStateTxFileOnly(scratch->sgh, f, tx->tx_ptr, tx->tx_id,
+                        flow_flags, file_no_match);
+            }
+        } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
+            TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
+                    "mpm won't trigger for it anymore");
+        } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
+            TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
+                    "mpm will revisit it");
+        } else {
+            TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
+            DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
+                    inspect_flags, flow_flags, file_no_match);
+        }
+    }
+
+    return retval;
+}
+
+/** \internal
+ *  \brief get a DetectTransaction object
+ *  \retval struct filled with relevant info or all nulls/0s
+ */
+static DetectTransaction GetTx(const uint8_t ipproto, const AppProto alproto,
+        void *alstate, const uint64_t tx_id, const int tx_end_state,
+        const uint8_t flow_flags)
+{
+    void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id);
+    if (tx_ptr == NULL) {
+        DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
+        return no_tx;
+    }
+    const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags);
+    if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
+        SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
+                tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
+                detect_flags);
+        DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
+        return no_tx;
+    }
+
+    const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
+    const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
+    DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(ipproto, alproto, tx_ptr);
+    DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
+    uint64_t prefilter_flags = detect_flags & APP_LAYER_TX_PREFILTER_MASK;
+
+    DetectTransaction tx = {
+                            .tx_ptr = tx_ptr,
+                            .tx_id = tx_id,
+                            .de_state = tx_dir_state,
+                            .detect_flags = detect_flags,
+                            .prefilter_flags = prefilter_flags,
+                            .prefilter_flags_orig = prefilter_flags,
+                            .tx_progress = tx_progress,
+                            .tx_end_state = tx_end_state,
+                           };
+    return tx;
+}
+
+static void DetectRunTx(ThreadVars *tv,
+                    DetectEngineCtx *de_ctx,
+                    DetectEngineThreadCtx *det_ctx,
+                    Packet *p,
+                    Flow *f,
+                    DetectRunScratchpad *scratch)
+{
+    const uint8_t flow_flags = scratch->flow_flags;
+    const SigGroupHead * const sgh = scratch->sgh;
+    void * const alstate = f->alstate;
+    const uint8_t ipproto = f->proto;
+    const AppProto alproto = f->alproto;
+
+    const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
+    uint64_t tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
+    const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
+
+    for ( ; tx_id < total_txs; tx_id++) {
+        DetectTransaction tx = GetTx(ipproto, alproto,
+                alstate, tx_id, tx_end_state, flow_flags);
+        if (tx.tx_ptr == NULL) {
+            SCLogDebug("%p/%"PRIu64" no transaction to inspect",
+                    tx.tx_ptr, tx_id);
+            continue;
+        }
+
+        uint32_t array_idx = 0;
+        uint32_t total_rules = det_ctx->match_array_cnt;
+        total_rules += (tx.de_state ? tx.de_state->cnt : 0);
+
+        /* run prefilter engines and merge results into a candidates array */
+        if (sgh->tx_engines) {
+            DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto,
+                    alstate, &tx);
+            SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates",
+                    tx.tx_ptr, tx_id, det_ctx->pmq.rule_id_array_cnt);
+
+            total_rules += det_ctx->pmq.rule_id_array_cnt;
+            if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
+                RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
+            }
+
+            for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) {
+                const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]];
+                const SigIntId id = s->num;
+                det_ctx->tx_candidates[array_idx].s = s;
+                det_ctx->tx_candidates[array_idx].id = id;
+                det_ctx->tx_candidates[array_idx].flags = NULL;
+                det_ctx->tx_candidates[array_idx].stream_reset = 0;
+                array_idx++;
+            }
+        } else {
+            if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) {
+                RuleMatchCandidateTxArrayExpand(det_ctx, total_rules);
+            }
+        }
+
+        /* merge 'state' rules from the regular prefilter */
+        uint32_t x = array_idx;
+        for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) {
+            const Signature *s = det_ctx->match_array[i];
+            if (s->flags & SIG_FLAG_STATE_MATCH) {
+                const SigIntId id = s->num;
+                det_ctx->tx_candidates[array_idx].s = s;
+                det_ctx->tx_candidates[array_idx].id = id;
+                det_ctx->tx_candidates[array_idx].flags = NULL;
+                det_ctx->tx_candidates[array_idx].stream_reset = 0;
+                array_idx++;
+
+                SCLogDebug("%p/%"PRIu64" rule %u (%u) added from 'match' list",
+                        tx.tx_ptr, tx_id, s->id, id);
+            }
+        }
+        SCLogDebug("%p/%"PRIu64" rules added from 'match' list: %u",
+                tx.tx_ptr, tx_id, array_idx - x); (void)x;
+
+        /* merge stored state into results */
+        if (tx.de_state != NULL) {
+            const uint32_t old = array_idx;
+
+            SigIntId state_cnt = 0;
+            DeStateStore *tx_store = tx.de_state->head;
+            for (; tx_store != NULL; tx_store = tx_store->next) {
+                SCLogDebug("tx_store %p", tx_store);
+
+                SigIntId store_cnt = 0;
+                for (store_cnt = 0;
+                        store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt;
+                        store_cnt++, state_cnt++)
+                {
+                    DeStateStoreItem *item = &tx_store->store[store_cnt];
+                    SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
+                    det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
+                    det_ctx->tx_candidates[array_idx].id = item->sid;
+                    det_ctx->tx_candidates[array_idx].flags = &item->flags;
+                    det_ctx->tx_candidates[array_idx].stream_reset = 0;
+                    array_idx++;
+                }
+            }
+            if (old && old != array_idx) {
+                qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx),
+                        DetectRunTxSortHelper);
+
+                SCLogDebug("%p/%"PRIu64" rules added from 'continue' list: %u",
+                        tx.tx_ptr, tx_id, array_idx - old);
+            }
+        }
+
+        det_ctx->tx_id = tx_id;
+        det_ctx->tx_id_set = 1;
+        det_ctx->p = p;
+
+        /* run rules: inspect the match candidates */
+        for (uint32_t i = 0; i < array_idx; i++) {
+            RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i];
+            const Signature *s = det_ctx->tx_candidates[i].s;
+            uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags;
+
+            /* deduplicate: rules_array is sorted, but not deduplicated:
+             * both mpm and stored state could give us the same sid.
+             * As they are back to back in that case we can check for it
+             * here. We select the stored state one. */
+            if ((i + 1) < array_idx) {
+                if (det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i+1].s) {
+                    if (det_ctx->tx_candidates[i].flags != NULL) {
+                        i++;
+                        SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
+                                tx.tx_ptr, tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
+                    } else if (det_ctx->tx_candidates[i+1].flags != NULL) {
+                        SCLogDebug("%p/%"PRIu64" inspecting SKIP CURRENT: sid %u (%u), flags %08x",
+                                tx.tx_ptr, tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
+                        continue;
+                    } else {
+                        // if it's all the same, inspect the current one and skip next.
+                        i++;
+                        SCLogDebug("%p/%"PRIu64" inspecting SKIP NEXT: sid %u (%u), flags %08x",
+                                tx.tx_ptr, tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
+                    }
+                }
+            }
+
+            SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x",
+                    tx.tx_ptr, tx_id, s->id, s->num, inspect_flags ? *inspect_flags : 0);
+
+            if (inspect_flags) {
+                if (*inspect_flags & (DE_STATE_FLAG_FULL_INSPECT|DE_STATE_FLAG_SIG_CANT_MATCH)) {
+                    SCLogDebug("%p/%"PRIu64" inspecting: sid %u (%u), flags %08x ALREADY COMPLETE",
+                            tx.tx_ptr, tx_id, s->id, s->num, *inspect_flags);
+                    continue;
+                }
+            }
+
+            if (inspect_flags) {
+                /* continue previous inspection */
+                SCLogDebug("%p/%"PRIu64" Continueing sid %u", tx.tx_ptr, tx_id, s->id);
+            } else {
+                /* start new inspection */
+                SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx_id, s->id);
+            }
+
+            /* call individual rule inspection */
+            const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
+                    alstate, &tx, s, inspect_flags, can, scratch);
+            if (r == 1) {
+                /* match */
+                DetectRunPostMatch(tv, det_ctx, p, s);
+
+                uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_TX);
+                if (s->action & ACTION_DROP)
+                    alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
+
+                SCLogDebug("%p/%"PRIu64" sig %u (%u) matched", tx.tx_ptr, tx_id, s->id, s->num);
+                if (!(s->flags & SIG_FLAG_NOALERT)) {
+                    PacketAlertAppend(det_ctx, s, p, tx_id, alert_flags);
+                } else {
+                    DetectSignatureApplyActions(p, s, alert_flags);
+                }
+            }
+            DetectVarProcessList(det_ctx, p->flow, p);
+        }
+
+        det_ctx->tx_id = 0;
+        det_ctx->tx_id_set = 0;
+        det_ctx->p = NULL;
+
+        /* see if we have any updated state to store in the tx */
+
+        uint64_t new_detect_flags = 0;
+        /* this side of the tx is done */
+        if (tx.tx_progress >= tx.tx_end_state) {
+            new_detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+            SCLogDebug("%p/%"PRIu64" tx is done for direction %s. Flag %016"PRIx64,
+                    tx.tx_ptr, tx_id,
+                    flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
+                    new_detect_flags);
+        }
+        if (tx.prefilter_flags != tx.prefilter_flags_orig) {
+            new_detect_flags |= tx.prefilter_flags;
+            SCLogDebug("%p/%"PRIu64" updated prefilter flags %016"PRIx64" "
+                    "(was: %016"PRIx64") for direction %s. Flag %016"PRIx64,
+                    tx.tx_ptr, tx_id, tx.prefilter_flags, tx.prefilter_flags_orig,
+                    flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",
+                    new_detect_flags);
+        }
+        if (new_detect_flags != 0 &&
+                (new_detect_flags | tx.detect_flags) != tx.detect_flags)
+        {
+            new_detect_flags |= tx.detect_flags;
+            SCLogDebug("%p/%"PRIu64" Storing new flags %016"PRIx64" (was %016"PRIx64")",
+                    tx.tx_ptr, tx_id, new_detect_flags, tx.detect_flags);
+            AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr,
+                    flow_flags, new_detect_flags);
+        }
+    }
+}
+
 /** \brief Apply action(s) and Set 'drop' sig info,
  *         if applicable */
 void DetectSignatureApplyActions(Packet *p,
@@ -1043,13 +1586,13 @@ static void DetectFlow(ThreadVars *tv,
                 flags = STREAM_TOCLIENT;
             }
             flags = FlowGetDisruptionFlags(p->flow, flags);
-            DeStateUpdateInspectTransactionId(p->flow, flags);
+            DeStateUpdateInspectTransactionId(p->flow, flags, true);
         }
         return;
     }
 
     /* see if the packet matches one or more of the sigs */
-    (void)SigMatchSignatures(tv,de_ctx,det_ctx,p);
+    (void)DetectRun(tv, de_ctx, det_ctx, p);
 }
 
 
@@ -1065,7 +1608,7 @@ static void DetectNoFlow(ThreadVars *tv,
     }
 
     /* see if the packet matches one or more of the sigs */
-    (void)SigMatchSignatures(tv,de_ctx,det_ctx,p);
+    DetectRun(tv, de_ctx, det_ctx, p);
     return;
 }
 
@@ -1141,6 +1684,18 @@ void DisableDetectFlowFileFlags(Flow *f)
     DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT);
 }
 
+#ifdef UNITTESTS
+/**
+ *  \brief wrapper for old tests
+ */
+void SigMatchSignatures(ThreadVars *th_v,
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        Packet *p)
+{
+    DetectRun(th_v, de_ctx, det_ctx, p);
+}
+#endif
+
 /*
  * TESTS
  */
diff --git a/src/detect.h b/src/detect.h
index a98cfcb15a..f260732445 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -260,21 +260,11 @@ typedef struct DetectPort_ {
 #define SIG_MASK_REQUIRE_FLAGS_INITDEINIT   (1<<2)    /* SYN, FIN, RST */
 #define SIG_MASK_REQUIRE_FLAGS_UNUSUAL      (1<<3)    /* URG, ECN, CWR */
 #define SIG_MASK_REQUIRE_NO_PAYLOAD         (1<<4)
-#define SIG_MASK_REQUIRE_HTTP_STATE         (1<<5)
-#define SIG_MASK_REQUIRE_DCE_STATE          (1<<6)
+//
 #define SIG_MASK_REQUIRE_ENGINE_EVENT       (1<<7)
-#define SIG_MASK_REQUIRE_SSH_STATE          (1<<8)
-#define SIG_MASK_REQUIRE_TLS_STATE          (1<<9)
-#define SIG_MASK_REQUIRE_DNS_STATE          (1<<10)
-#define SIG_MASK_REQUIRE_FTP_STATE          (1<<11)
-#define SIG_MASK_REQUIRE_FTPDATA_STATE      (1<<12)
-#define SIG_MASK_REQUIRE_SMTP_STATE         (1<<13)
-#define SIG_MASK_REQUIRE_TEMPLATE_STATE     (1<<14)
-#define SIG_MASK_REQUIRE_ENIP_STATE         (1<<15)
-#define SIG_MASK_REQUIRE_DNP3_STATE         (1<<16)
 
 /* for now a uint8_t is enough */
-#define SignatureMask uint32_t
+#define SignatureMask uint8_t
 
 #define DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH 0x0004
 
@@ -342,7 +332,8 @@ typedef struct DetectEngineAppInspectionEngine_ {
     uint8_t dir;
     uint8_t id;     /**< per sig id used in state keeping */
     uint16_t mpm:1;
-    uint16_t sm_list:15;
+    uint16_t stream:1;
+    uint16_t sm_list:14;
     int16_t progress;
 
     /* \retval 0 No match.  Don't discontinue matching yet.  We need more data.
@@ -809,8 +800,24 @@ typedef struct FiledataReassembledBody_ {
 typedef struct SignatureNonPrefilterStore_ {
     SigIntId id;
     SignatureMask mask;
+    uint8_t alproto;
 } SignatureNonPrefilterStore;
 
+/** array of TX inspect rule candidates */
+typedef struct RuleMatchCandidateTx {
+    SigIntId id;            /**< internal signature id */
+    uint32_t *flags;        /**< inspect flags ptr */
+    union {
+        struct {
+            bool stream_stored;
+            uint8_t stream_result;
+        };
+        uint32_t stream_reset;
+    };
+
+    const Signature *s;     /**< ptr to sig */
+} RuleMatchCandidateTx;
+
 /**
   * Detection engine thread data.
   */
@@ -883,8 +890,6 @@ typedef struct DetectEngineThreadCtx_ {
     /** ID of the transaction currently being inspected. */
     uint64_t tx_id;
     Packet *p;
-    bool stream_already_inspected;
-    int stream_last_result;
 
     SC_ATOMIC_DECLARE(int, so_far_used_by_detect);
 
@@ -900,11 +905,8 @@ typedef struct DetectEngineThreadCtx_ {
     /** size in use */
     SigIntId match_array_cnt;
 
-    /** Array of sigs that had a state change */
-    SigIntId de_state_sig_array_len;
-    uint8_t *de_state_sig_array;
-
-    const struct SigGroupHead_ *sgh;
+    RuleMatchCandidateTx *tx_candidates;
+    uint32_t tx_candidates_size;
 
     SignatureNonPrefilterStore *non_pf_store_ptr;
     uint32_t non_pf_store_cnt;
@@ -1245,11 +1247,11 @@ Signature *DetectGetTagSignature(void);
 int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int);
 void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *, int);
 
-int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
-                                   DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
-                                   const Signature *s);
 void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t);
 
+void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size);
+void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx);
+
 void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx);
 
 #include "detect-engine-build.h"