detect/file-data: Improved support for share bufs

This commit improves support for shared buffer usage, i.e., when
multiple rules share the file data (http) buffer and apply different
combinations of transforms and fast_patterns (or none).

diff --git a/src/detect-file-data.c b/src/detect-file-data.c
index 13246ee1ebc23f1db9174ecc0e5c8982d251c5c2..d34f5260b9be6082a504138fa01f319546d42080 100644 (file)
--- a/src/detect-file-data.c
+++ b/src/detect-file-data.c
@@ -59,6 +59,8 @@ static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
                                         Signature *s);
 static int g_file_data_buffer_id = 0;
 
+static inline HtpBody *GetResponseBody(htp_tx_t *tx);
+
 /* HTTP */
 static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms,
@@ -75,6 +77,10 @@ int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
         const DetectBufferMpmRegistery *mpm_reg, int list_id);
 
+static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
+        const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
+
 /**
  * \brief Registration function for keyword: file_data
  */
@@ -110,9 +116,8 @@ void DetectFiledataRegister(void)
             PrefilterMpmFiledataRegister, NULL,
             ALPROTO_HTTP2, HTTP2StateDataServer);
 
-    DetectAppLayerInspectEngineRegister2("file_data",
-            ALPROTO_HTTP, SIG_FLAG_TOCLIENT, HTP_RESPONSE_BODY,
-            DetectEngineInspectBufferGeneric, HttpServerBodyGetDataCallback);
+    DetectAppLayerInspectEngineRegister2("file_data", ALPROTO_HTTP, SIG_FLAG_TOCLIENT,
+            HTP_RESPONSE_BODY, DetectEngineInspectBufferHttpBody, HttpServerBodyGetDataCallback);
     DetectAppLayerInspectEngineRegister2("file_data",
             ALPROTO_SMTP, SIG_FLAG_TOSERVER, 0,
             DetectEngineInspectFiledata, NULL);
@@ -162,6 +167,73 @@ static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
     de_ctx->filedata_config_initialized = true;
 }
 
+static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
+        const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
+{
+    const int list_id = engine->sm_list;
+    const InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    bool eof = false;
+    if (buffer->inspect == NULL) {
+        SCLogDebug("running inspect on %d", list_id);
+
+        eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
+
+        SCLogDebug("list %d mpm? %s transforms %p", engine->sm_list, engine->mpm ? "true" : "false",
+                engine->v2.transforms);
+
+        /* if prefilter didn't already run, we need to consider transformations */
+        const DetectEngineTransforms *transforms = NULL;
+        if (!engine->mpm) {
+            transforms = engine->v2.transforms;
+        }
+
+        buffer = engine->v2.GetData(det_ctx, transforms, f, flags, txv, list_id);
+        if (unlikely(buffer == NULL)) {
+            return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
+    }
+
+    const uint32_t data_len = buffer->inspect_len;
+    const uint8_t *data = buffer->inspect;
+    const uint64_t offset = buffer->inspect_offset;
+
+    uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
+    ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
+    ci_flags |= buffer->flags;
+
+    det_ctx->discontinue_matching = 0;
+    det_ctx->buffer_offset = 0;
+    det_ctx->inspection_recursion_counter = 0;
+
+    /* Inspect all the uricontents fetched on each
+     * transaction at the app layer */
+    int r = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, (uint8_t *)data,
+            data_len, offset, ci_flags, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
+
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    htp_tx_t *tx = txv;
+    HtpBody *body = GetResponseBody(tx);
+    body->body_inspected = body->content_len_so_far;
+    SCLogDebug("body->body_inspected now: %" PRIu64, body->body_inspected);
+
+    if (r == 1) {
+        return DETECT_ENGINE_INSPECT_SIG_MATCH;
+    }
+
+    if (flags & STREAM_TOSERVER) {
+        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, txv, flags) >
+                HTP_REQUEST_BODY)
+            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+    } else {
+        if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, txv, flags) >
+                HTP_RESPONSE_BODY)
+            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+    }
+    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+}
+
 /**
  * \brief this function is used to parse filedata options
  * \brief into the current signature
@@ -336,11 +408,6 @@ static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *de
 
     InspectionBufferApplyTransforms(buffer, transforms);
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
-     * the window sizes when freeing data */
-    body->body_inspected = body->content_len_so_far;
-    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
-
     SCReturnPtr(buffer, "InspectionBuffer");
 }
 
@@ -367,10 +434,11 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
     // TODO this is unused, is that right?
     //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
 
-    SCLogDebug("content_limit %u, content_inspect_min_size %u",
-                content_limit, content_inspect_min_size);
+    SCLogDebug("[list %d] first: %d, content_limit %u, content_inspect_min_size %u", list_id,
+            first ? 1 : 0, content_limit, content_inspect_min_size);
 
-    SCLogDebug("file %p size %"PRIu64", state %d", cur_file, file_size, cur_file->state);
+    SCLogDebug("[list %d] file %p size %" PRIu64 ", state %d", list_id, cur_file, file_size,
+            cur_file->state);
 
     /* no new data */
     if (cur_file->content_inspected == file_size) {
@@ -399,15 +467,18 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
             &data, &data_len,
             cur_file->content_inspected);
     InspectionBufferSetup(buffer, data, data_len);
+    SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
+               "; data_len %" PRIu32 "; file_size %" PRIu64,
+            list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
     buffer->inspect_offset = cur_file->content_inspected;
     InspectionBufferApplyTransforms(buffer, transforms);
+    SCLogDebug("[list %d] [after] buffer offset %" PRIu64 "; buffer len %" PRIu32, list_id,
+            buffer->inspect_offset, buffer->inspect_len);
 
-    /* update inspected tracker */
-    cur_file->content_inspected = file_size;
-    SCLogDebug("content_inspected %"PRIu64, cur_file->content_inspected);
+    SCLogDebug("[list %d] content_inspected %" PRIu64, list_id, cur_file->content_inspected);
 
-    SCLogDebug("file_data buffer %p, data %p len %u offset %"PRIu64,
-        buffer, buffer->inspect, buffer->inspect_len, buffer->inspect_offset);
+    SCLogDebug("[list %d] file_data buffer %p, data %p len %u offset %" PRIu64, list_id, buffer,
+            buffer->inspect, buffer->inspect_len, buffer->inspect_offset);
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }
@@ -456,6 +527,8 @@ static int DetectEngineInspectFiledata(
                                               buffer->inspect_len,
                                               buffer->inspect_offset, ciflags,
                                               DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
+        /* update inspected tracker */
+        file->content_inspected = buffer->inspect_len + buffer->inspect_offset;
         if (match == 1) {
             r = 1;
             break;