]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
detect/file-data: consolidate matching code
authorVictor Julien <victor@inliniac.net>
Thu, 13 Dec 2018 12:44:17 +0000 (13:44 +0100)
committerVictor Julien <victor@inliniac.net>
Tue, 29 Jan 2019 12:27:57 +0000 (13:27 +0100)
src/detect-engine-filedata.c
src/detect-engine-filedata.h
src/detect-engine-hsbd.c
src/detect-engine-hsbd.h
src/detect-file-data.c
src/tests/detect.c

index d8d3721042adb95ed6bd8d4a35447d4f09daa58e..e9e3bb07292def35670b642f61150a416cef1341 100644 (file)
 
 #include "app-layer-parser.h"
 
-static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
-        const DetectEngineTransforms *transforms,
-        Flow *f, uint8_t flow_flags, File *cur_file,
-        int list_id, int local_file_id, bool first)
-{
-    SCEnter();
-
-    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
-    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, local_file_id);
-    if (buffer == NULL)
-        return NULL;
-    if (!first && buffer->inspect != NULL)
-        return buffer;
-
-    const uint64_t file_size = FileDataSize(cur_file);
-    const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
-    const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
-    const uint32_t content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
-    // TODO this is unused, is that right?
-    //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
-
-    SCLogDebug("content_limit %u, content_inspect_min_size %u",
-                content_limit, content_inspect_min_size);
-
-    SCLogDebug("file %p size %"PRIu64", state %d", cur_file, file_size, cur_file->state);
-
-    /* no new data */
-    if (cur_file->content_inspected == file_size) {
-        SCLogDebug("no new data");
-        return NULL;
-    }
-
-    if (file_size == 0) {
-        SCLogDebug("no data to inspect for this transaction");
-        return NULL;
-    }
-
-    if ((content_limit == 0 || file_size < content_limit) &&
-        file_size < content_inspect_min_size &&
-        !(flow_flags & STREAM_EOF) && !(cur_file->state > FILE_STATE_OPENED)) {
-        SCLogDebug("we still haven't seen the entire content. "
-                   "Let's defer content inspection till we see the "
-                   "entire content.");
-        return NULL;
-    }
-
-    const uint8_t *data;
-    uint32_t data_len;
-
-    StreamingBufferGetDataAtOffset(cur_file->sb,
-            &data, &data_len,
-            cur_file->content_inspected);
-    InspectionBufferSetup(buffer, data, data_len);
-    buffer->inspect_offset = cur_file->content_inspected;
-    InspectionBufferApplyTransforms(buffer, transforms);
-
-    /* update inspected tracker */
-    cur_file->content_inspected = file_size;
-    SCLogDebug("content_inspected %"PRIu64, cur_file->content_inspected);
-
-    SCLogDebug("file_data buffer %p, data %p len %u offset %"PRIu64,
-        buffer, buffer->inspect, buffer->inspect_len, buffer->inspect_offset);
-
-    SCReturnPtr(buffer, "InspectionBuffer");
-}
-
-int DetectEngineInspectFiledata(
-        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
-        const DetectEngineAppInspectionEngine *engine,
-        const Signature *s,
-        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
-{
-    int r = 0;
-    int match = 0;
-
-    // TODO remove
-    if (f->alproto == ALPROTO_HTTP) {
-        abort();
-    }
-
-    const DetectEngineTransforms *transforms = NULL;
-    if (!engine->mpm) {
-        transforms = engine->v2.transforms;
-    }
-
-    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto,
-                                                f->alstate, flags);
-    if (ffc == NULL) {
-        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
-    }
-
-    int local_file_id = 0;
-    File *file = ffc->head;
-    for (; file != NULL; file = file->next) {
-        if (file->txid != tx_id)
-            continue;
-
-        InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx,
-            transforms, f, flags, file, engine->sm_list, local_file_id, false);
-        if (buffer == NULL)
-            continue;
-
-        bool eof = (file->state == FILE_STATE_CLOSED);
-        uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0;
-        if (buffer->inspect_offset == 0)
-            ciflags |= DETECT_CI_FLAGS_START;
-
-        det_ctx->buffer_offset = 0;
-        det_ctx->discontinue_matching = 0;
-        det_ctx->inspection_recursion_counter = 0;
-        match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd,
-                                              f,
-                                              (uint8_t *)buffer->inspect,
-                                              buffer->inspect_len,
-                                              buffer->inspect_offset, ciflags,
-                                              DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
-        if (match == 1) {
-            r = 1;
-            break;
-        }
-        local_file_id++;
-    }
-
-    if (r == 1)
-        return DETECT_ENGINE_INSPECT_SIG_MATCH;
-    else
-        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
-}
-
-typedef struct PrefilterMpmFiledata {
-    int list_id;
-    const MpmCtx *mpm_ctx;
-    const DetectEngineTransforms *transforms;
-} PrefilterMpmFiledata;
-
-/** \brief Filedata Filedata Mpm prefilter callback
- *
- *  \param det_ctx detection engine thread ctx
- *  \param p packet to inspect
- *  \param f flow to inspect
- *  \param txv tx to inspect
- *  \param pectx inspection context
- */
-static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
-        const void *pectx,
-        Packet *p, Flow *f, void *txv,
-        const uint64_t idx, const uint8_t flags)
-{
-    SCEnter();
-
-    const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
-    const MpmCtx *mpm_ctx = ctx->mpm_ctx;
-    const int list_id = ctx->list_id;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto,
-                                                f->alstate, flags);
-    int local_file_id = 0;
-    if (ffc != NULL) {
-        File *file = ffc->head;
-        for (; file != NULL; file = file->next) {
-            if (file->txid != idx)
-                continue;
-
-            InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx,
-                    ctx->transforms, f, flags, file, list_id, local_file_id, true);
-            if (buffer == NULL)
-                continue;
-
-            if (buffer->inspect_len >= mpm_ctx->minlen) {
-                (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                        &det_ctx->mtcu, &det_ctx->pmq,
-                        buffer->inspect, buffer->inspect_len);
-            }
-        }
-    }
-}
-
-static void PrefilterMpmFiledataFree(void *ptr)
-{
-    SCFree(ptr);
-}
-
-int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
-{
-    PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
-    if (pectx == NULL)
-        return -1;
-    pectx->list_id = list_id;
-    pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
-
-    return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
-            pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
-}
-
 #ifdef UNITTESTS
 #include "tests/detect-engine-filedata.c"
 #endif /* UNITTESTS */
index 6e3d21f60c1d60d18e9ce005803cb2eb85eb10cd..0f30d50ccb32db68a2cf436847619debe9573b02 100644 (file)
 #ifndef __DETECT_ENGINE_FILEDATA_H__
 #define __DETECT_ENGINE_FILEDATA_H__
 
-int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
-int DetectEngineInspectFiledata(
-        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
-        const DetectEngineAppInspectionEngine *engine,
-        const Signature *s,
-        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
-
-void DetectEngineCleanFiledataBuffers(DetectEngineThreadCtx *det_ctx);
-
 void DetectEngineSMTPFiledataRegisterTests(void);
 
 #endif /* __DETECT_ENGINE_FILEDATA_H__ */
index 5b1e0cb96668d4733e0541061c90f06f3c95e9e8..6c18f4f63ffc67a0f1464f0d2f9b3cd1ca654616 100644 (file)
 
 #include "util-validate.h"
 
-static inline HtpBody *GetResponseBody(htp_tx_t *tx)
-{
-    HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
-    if (htud == NULL) {
-        SCLogDebug("no htud");
-        return NULL;
-    }
-
-    return &htud->response_body;
-}
-
-InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
-        const DetectEngineTransforms *transforms,
-        Flow *f, const uint8_t flow_flags,
-        void *txv, const int list_id)
-{
-    SCEnter();
-
-    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
-    if (buffer->inspect != NULL)
-        return buffer;
-
-    htp_tx_t *tx = txv;
-    HtpState *htp_state = f->alstate;
-    const uint8_t flags = flow_flags;
-
-    HtpBody *body = GetResponseBody(tx);
-    if (body == NULL) {
-        return NULL;
-    }
-
-    /* no new data */
-    if (body->body_inspected == body->content_len_so_far) {
-        SCLogDebug("no new data");
-        return NULL;
-    }
-
-    HtpBodyChunk *cur = body->first;
-    if (cur == NULL) {
-        SCLogDebug("No http chunks to inspect for this transacation");
-        return NULL;
-    }
-
-    SCLogDebug("response.body_limit %u response_body.content_len_so_far %"PRIu64
-               ", response.inspect_min_size %"PRIu32", EOF %s, progress > body? %s",
-              htp_state->cfg->response.body_limit,
-              body->content_len_so_far,
-              htp_state->cfg->response.inspect_min_size,
-              flags & STREAM_EOF ? "true" : "false",
-               (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
-
-    if (!htp_state->cfg->http_body_inline) {
-        /* inspect the body if the transfer is complete or we have hit
-        * our body size limit */
-        if ((htp_state->cfg->response.body_limit == 0 ||
-             body->content_len_so_far < htp_state->cfg->response.body_limit) &&
-            body->content_len_so_far < htp_state->cfg->response.inspect_min_size &&
-            !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) &&
-            !(flags & STREAM_EOF)) {
-            SCLogDebug("we still haven't seen the entire response body.  "
-                       "Let's defer body inspection till we see the "
-                       "entire body.");
-            return NULL;
-        }
-    }
-
-    /* get the inspect buffer
-     *
-     * make sure that we have at least the configured inspect_win size.
-     * If we have more, take at least 1/4 of the inspect win size before
-     * the new data.
-     */
-    uint64_t offset = 0;
-    if (body->body_inspected > htp_state->cfg->response.inspect_min_size) {
-        BUG_ON(body->content_len_so_far < body->body_inspected);
-        uint64_t inspect_win = body->content_len_so_far - body->body_inspected;
-        SCLogDebug("inspect_win %"PRIu64, inspect_win);
-        if (inspect_win < htp_state->cfg->response.inspect_window) {
-            uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
-            if (body->body_inspected < inspect_short)
-                offset = 0;
-            else
-                offset = body->body_inspected - inspect_short;
-        } else {
-            offset = body->body_inspected - (htp_state->cfg->response.inspect_window / 4);
-        }
-    }
-
-    const uint8_t *data;
-    uint32_t data_len;
-
-    StreamingBufferGetDataAtOffset(body->sb,
-            &data, &data_len, offset);
-    InspectionBufferSetup(buffer, data, data_len);
-    buffer->inspect_offset = offset;
-
-    /* built-in 'transformation' */
-    if (htp_state->cfg->swf_decompression_enabled) {
-        int swf_file_type = FileIsSwfFile(data, data_len);
-        if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION ||
-            swf_file_type == FILE_SWF_LZMA_COMPRESSION)
-        {
-            (void)FileSwfDecompression(data, data_len,
-                                       det_ctx,
-                                       buffer,
-                                       htp_state->cfg->swf_compression_type,
-                                       htp_state->cfg->swf_decompress_depth,
-                                       htp_state->cfg->swf_compress_depth);
-        }
-    }
-
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    body->body_inspected = body->content_len_so_far;
-    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
-
-    SCReturnPtr(buffer, "InspectionBuffer");
-}
 
 /***********************************Unittests**********************************/
 
index 060269b82bb49dbb29a9482d96bbc505bdb92a6c..71a7d745a7325166cce8136320437653a8b4ac50 100644 (file)
 #ifndef __DETECT_ENGINE_HSBD_H__
 #define __DETECT_ENGINE_HSBD_H__
 
-#include "app-layer-htp.h"
-
-int PrefilterTxHttpResponseBodyRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx);
-
-int DetectEngineInspectHttpServerBody(ThreadVars *tv,
-        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
-        const Signature *s, const SigMatchData *smd,
-        Flow *f, uint8_t flags, void *alstate,
-        void *tx, uint64_t tx_id);
-
-InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
-        const DetectEngineTransforms *transform,
-        Flow *f, const uint8_t flow_flags, void *txv, const int list_id);
-
-void DetectEngineCleanHSBDBuffers(DetectEngineThreadCtx *det_ctx);
-
 void DetectEngineHttpServerBodyRegisterTests(void);
 
 #endif /* __DETECT_ENGINE_HSBD_H__ */
index 39686ec3d78b4eccbf92b9e90ea544a7bafa7e5f..f2893eb7e44da1e32006ea199298ffd1c7d433f0 100644 (file)
 #include "detect-engine-mpm.h"
 #include "detect-engine-state.h"
 #include "detect-engine-prefilter.h"
+#include "detect-engine-content-inspection.h"
 #include "detect-engine-filedata.h"
 #include "detect-engine-hsbd.h"
 #include "detect-file-data.h"
 
+#include "app-layer-parser.h"
+#include "app-layer-htp.h"
 #include "app-layer-smtp.h"
 
 #include "flow.h"
@@ -48,6 +51,7 @@
 #include "util-spm-bm.h"
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
+#include "util-file-decompression.h"
 
 static int DetectFiledataSetup (DetectEngineCtx *, Signature *, const char *);
 static void DetectFiledataRegisterTests(void);
@@ -55,6 +59,22 @@ static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
                                         Signature *s);
 static int g_file_data_buffer_id = 0;
 
+/* HTTP */
+static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms,
+        Flow *f, const uint8_t flow_flags,
+        void *txv, const int list_id);
+
+/* file API */
+static int DetectEngineInspectFiledata(
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        const DetectEngineAppInspectionEngine *engine,
+        const Signature *s,
+        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
+int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
+        SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+
 /**
  * \brief Registration function for keyword: file_data
  */
@@ -187,6 +207,327 @@ static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
     SCLogDebug("callback invoked by %u", s->id);
 }
 
+/* HTTP based detection */
+
+static inline HtpBody *GetResponseBody(htp_tx_t *tx)
+{
+    HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (htud == NULL) {
+        SCLogDebug("no htud");
+        return NULL;
+    }
+
+    return &htud->response_body;
+}
+
+static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms,
+        Flow *f, const uint8_t flow_flags,
+        void *txv, const int list_id)
+{
+    SCEnter();
+
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    if (buffer->inspect != NULL)
+        return buffer;
+
+    htp_tx_t *tx = txv;
+    HtpState *htp_state = f->alstate;
+    const uint8_t flags = flow_flags;
+
+    HtpBody *body = GetResponseBody(tx);
+    if (body == NULL) {
+        return NULL;
+    }
+
+    /* no new data */
+    if (body->body_inspected == body->content_len_so_far) {
+        SCLogDebug("no new data");
+        return NULL;
+    }
+
+    HtpBodyChunk *cur = body->first;
+    if (cur == NULL) {
+        SCLogDebug("No http chunks to inspect for this transacation");
+        return NULL;
+    }
+
+    SCLogDebug("response.body_limit %u response_body.content_len_so_far %"PRIu64
+               ", response.inspect_min_size %"PRIu32", EOF %s, progress > body? %s",
+              htp_state->cfg->response.body_limit,
+              body->content_len_so_far,
+              htp_state->cfg->response.inspect_min_size,
+              flags & STREAM_EOF ? "true" : "false",
+               (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
+
+    if (!htp_state->cfg->http_body_inline) {
+        /* inspect the body if the transfer is complete or we have hit
+        * our body size limit */
+        if ((htp_state->cfg->response.body_limit == 0 ||
+             body->content_len_so_far < htp_state->cfg->response.body_limit) &&
+            body->content_len_so_far < htp_state->cfg->response.inspect_min_size &&
+            !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) &&
+            !(flags & STREAM_EOF)) {
+            SCLogDebug("we still haven't seen the entire response body.  "
+                       "Let's defer body inspection till we see the "
+                       "entire body.");
+            return NULL;
+        }
+    }
+
+    /* get the inspect buffer
+     *
+     * make sure that we have at least the configured inspect_win size.
+     * If we have more, take at least 1/4 of the inspect win size before
+     * the new data.
+     */
+    uint64_t offset = 0;
+    if (body->body_inspected > htp_state->cfg->response.inspect_min_size) {
+        BUG_ON(body->content_len_so_far < body->body_inspected);
+        uint64_t inspect_win = body->content_len_so_far - body->body_inspected;
+        SCLogDebug("inspect_win %"PRIu64, inspect_win);
+        if (inspect_win < htp_state->cfg->response.inspect_window) {
+            uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
+            if (body->body_inspected < inspect_short)
+                offset = 0;
+            else
+                offset = body->body_inspected - inspect_short;
+        } else {
+            offset = body->body_inspected - (htp_state->cfg->response.inspect_window / 4);
+        }
+    }
+
+    const uint8_t *data;
+    uint32_t data_len;
+
+    StreamingBufferGetDataAtOffset(body->sb,
+            &data, &data_len, offset);
+    InspectionBufferSetup(buffer, data, data_len);
+    buffer->inspect_offset = offset;
+
+    /* built-in 'transformation' */
+    if (htp_state->cfg->swf_decompression_enabled) {
+        int swf_file_type = FileIsSwfFile(data, data_len);
+        if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION ||
+            swf_file_type == FILE_SWF_LZMA_COMPRESSION)
+        {
+            (void)FileSwfDecompression(data, data_len,
+                                       det_ctx,
+                                       buffer,
+                                       htp_state->cfg->swf_compression_type,
+                                       htp_state->cfg->swf_decompress_depth,
+                                       htp_state->cfg->swf_compress_depth);
+        }
+    }
+
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    body->body_inspected = body->content_len_so_far;
+    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
+
+    SCReturnPtr(buffer, "InspectionBuffer");
+}
+
+/* file API based inspection */
+
+static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms,
+        Flow *f, uint8_t flow_flags, File *cur_file,
+        int list_id, int local_file_id, bool first)
+{
+    SCEnter();
+
+    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, local_file_id);
+    if (buffer == NULL)
+        return NULL;
+    if (!first && buffer->inspect != NULL)
+        return buffer;
+
+    const uint64_t file_size = FileDataSize(cur_file);
+    const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
+    const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
+    const uint32_t content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
+    // TODO this is unused, is that right?
+    //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
+
+    SCLogDebug("content_limit %u, content_inspect_min_size %u",
+                content_limit, content_inspect_min_size);
+
+    SCLogDebug("file %p size %"PRIu64", state %d", cur_file, file_size, cur_file->state);
+
+    /* no new data */
+    if (cur_file->content_inspected == file_size) {
+        SCLogDebug("no new data");
+        return NULL;
+    }
+
+    if (file_size == 0) {
+        SCLogDebug("no data to inspect for this transaction");
+        return NULL;
+    }
+
+    if ((content_limit == 0 || file_size < content_limit) &&
+        file_size < content_inspect_min_size &&
+        !(flow_flags & STREAM_EOF) && !(cur_file->state > FILE_STATE_OPENED)) {
+        SCLogDebug("we still haven't seen the entire content. "
+                   "Let's defer content inspection till we see the "
+                   "entire content.");
+        return NULL;
+    }
+
+    const uint8_t *data;
+    uint32_t data_len;
+
+    StreamingBufferGetDataAtOffset(cur_file->sb,
+            &data, &data_len,
+            cur_file->content_inspected);
+    InspectionBufferSetup(buffer, data, data_len);
+    buffer->inspect_offset = cur_file->content_inspected;
+    InspectionBufferApplyTransforms(buffer, transforms);
+
+    /* update inspected tracker */
+    cur_file->content_inspected = file_size;
+    SCLogDebug("content_inspected %"PRIu64, cur_file->content_inspected);
+
+    SCLogDebug("file_data buffer %p, data %p len %u offset %"PRIu64,
+        buffer, buffer->inspect, buffer->inspect_len, buffer->inspect_offset);
+
+    SCReturnPtr(buffer, "InspectionBuffer");
+}
+
+static int DetectEngineInspectFiledata(
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        const DetectEngineAppInspectionEngine *engine,
+        const Signature *s,
+        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
+{
+    int r = 0;
+    int match = 0;
+
+    // TODO remove
+    if (f->alproto == ALPROTO_HTTP) {
+        abort();
+    }
+
+    const DetectEngineTransforms *transforms = NULL;
+    if (!engine->mpm) {
+        transforms = engine->v2.transforms;
+    }
+
+    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto,
+                                                f->alstate, flags);
+    if (ffc == NULL) {
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
+
+    int local_file_id = 0;
+    File *file = ffc->head;
+    for (; file != NULL; file = file->next) {
+        if (file->txid != tx_id)
+            continue;
+
+        InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx,
+            transforms, f, flags, file, engine->sm_list, local_file_id, false);
+        if (buffer == NULL)
+            continue;
+
+        bool eof = (file->state == FILE_STATE_CLOSED);
+        uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0;
+        if (buffer->inspect_offset == 0)
+            ciflags |= DETECT_CI_FLAGS_START;
+
+        det_ctx->buffer_offset = 0;
+        det_ctx->discontinue_matching = 0;
+        det_ctx->inspection_recursion_counter = 0;
+        match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd,
+                                              f,
+                                              (uint8_t *)buffer->inspect,
+                                              buffer->inspect_len,
+                                              buffer->inspect_offset, ciflags,
+                                              DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
+        if (match == 1) {
+            r = 1;
+            break;
+        }
+        local_file_id++;
+    }
+
+    if (r == 1)
+        return DETECT_ENGINE_INSPECT_SIG_MATCH;
+    else
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+}
+
+typedef struct PrefilterMpmFiledata {
+    int list_id;
+    const MpmCtx *mpm_ctx;
+    const DetectEngineTransforms *transforms;
+} PrefilterMpmFiledata;
+
+/** \brief Filedata Filedata Mpm prefilter callback
+ *
+ *  \param det_ctx detection engine thread ctx
+ *  \param p packet to inspect
+ *  \param f flow to inspect
+ *  \param txv tx to inspect
+ *  \param pectx inspection context
+ */
+static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
+        const void *pectx,
+        Packet *p, Flow *f, void *txv,
+        const uint64_t idx, const uint8_t flags)
+{
+    SCEnter();
+
+    const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
+    const MpmCtx *mpm_ctx = ctx->mpm_ctx;
+    const int list_id = ctx->list_id;
+
+    FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto,
+                                                f->alstate, flags);
+    int local_file_id = 0;
+    if (ffc != NULL) {
+        File *file = ffc->head;
+        for (; file != NULL; file = file->next) {
+            if (file->txid != idx)
+                continue;
+
+            InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx,
+                    ctx->transforms, f, flags, file, list_id, local_file_id, true);
+            if (buffer == NULL)
+                continue;
+
+            if (buffer->inspect_len >= mpm_ctx->minlen) {
+                (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
+                        &det_ctx->mtcu, &det_ctx->pmq,
+                        buffer->inspect, buffer->inspect_len);
+            }
+        }
+    }
+}
+
+static void PrefilterMpmFiledataFree(void *ptr)
+{
+    SCFree(ptr);
+}
+
+int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
+        SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+{
+    PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
+    if (pectx == NULL)
+        return -1;
+    pectx->list_id = list_id;
+    pectx->mpm_ctx = mpm_ctx;
+    pectx->transforms = &mpm_reg->v2.transforms;
+
+    return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
+            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
+}
+
 #ifdef UNITTESTS
 #include "detect-isdataat.h"
 
index 087ead6bda0444cb73012fbd916151e46e149a01..81fc254cf7a3e6c1ecf4cb0b8262e5ccddb049fe 100644 (file)
@@ -17,6 +17,7 @@
 
 #ifdef UNITTESTS
 
+#include "../app-layer-htp.h"
 #include "../conf-yaml-loader.h"
 #include "../detect-parse.h"
 #include "../detect-engine-content-inspection.h"