detect/frame: get data using stream callback

Inspect only data that has already been consumed by the
app-layer parser. This allows for simpler progress tracking.

diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
index 71f6247e5c6d33b73ef926e0283dc24e0572773a..97ef816e795acdbb9193a1bd21cbdb0ebd695d56 100644 (file)
--- a/src/app-layer-frames.c
+++ b/src/app-layer-frames.c
@@ -224,23 +224,6 @@ static inline uint64_t FrameLeftEdge(
         }
     }
 }
-#if 0
-static inline uint64_t FramesLeftEdge(const TcpStream *stream, const Frames *frames)
-{
-    uint64_t le = STREAM_APP_PROGRESS(stream);
-    for (uint16_t i = 0; i < frames->cnt; i++) {
-        if (i < FRAMES_STATIC_CNT) {
-            const Frame *frame = &frames->sframes[i];
-            le = MIN(le, FrameLeftEdge(stream, frame));
-        } else {
-            const uint16_t o = i - FRAMES_STATIC_CNT;
-            const Frame *frame = &frames->dframes[o];
-            le = MIN(le, FrameLeftEdge(stream, frame));
-        }
-    }
-    return le;
-}
-#endif
 
 /** Stream buffer slides forward, we need to update and age out
  *  frame offsets/frames. Aging out means we move existing frames
@@ -274,12 +257,6 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c
     BUG_ON(frames == NULL);
     SCLogDebug("%s frames %p: sliding %u bytes", ds, frames, slide);
     uint64_t le = STREAM_APP_PROGRESS(stream);
-
-    if (slide >= frames->progress_rel)
-        frames->progress_rel = 0;
-    else
-        frames->progress_rel -= slide;
-
     const uint64_t next_base = STREAM_BASE_OFFSET(stream) + slide;
     const uint16_t start = frames->cnt;
     uint16_t removed = 0;
@@ -348,24 +325,6 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c
     return 0;
 }
 
-void AppLayerFramesUpdateProgress(
-        Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction)
-{
-    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
-    if (frames_container == NULL)
-        return;
-
-    Frames *frames;
-    if (direction == STREAM_TOSERVER) {
-        frames = &frames_container->toserver;
-    } else {
-        frames = &frames_container->toclient;
-    }
-
-    const uint32_t slide = progress - STREAM_APP_PROGRESS(stream);
-    frames->progress_rel += slide;
-}
-
 void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
 {
     FramesContainer *frames_container = AppLayerFramesGetContainer(f);
@@ -711,7 +670,7 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
     SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64,
             (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
             STREAM_BASE_OFFSET(stream));
-    const uint64_t abs_offset = STREAM_BASE_OFFSET(stream) + (uint64_t)frames->progress_rel;
+    const uint64_t abs_offset = STREAM_BASE_OFFSET(stream);
     const uint64_t acked = StreamTcpGetUsable(stream, eof);
     uint64_t le = STREAM_APP_PROGRESS(stream);
 

diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
index be2900224eef42fb42e390664433f9f7b7c29f08..746c7296926cba6fd2f1338a1ee71b144aa1e070 100644 (file)
--- a/src/app-layer-frames.h
+++ b/src/app-layer-frames.h
@@ -61,9 +61,8 @@ typedef struct Frame {
 typedef struct Frames {
     uint16_t cnt;
     uint16_t dyn_size;     /**< size in elements of `dframes` */
-    uint32_t progress_rel; /**< processing depth relative to STREAM_BASE_OFFSET */
-    uint64_t base_id;
     uint32_t left_edge_rel;
+    uint64_t base_id;
     Frame sframes[FRAMES_STATIC_CNT]; /**< static frames */
     Frame *dframes;                   /**< dynamically allocated space for more frames */
 #ifdef DEBUG
@@ -102,8 +101,6 @@ void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_
 void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id);
 void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id);
 
-void AppLayerFramesUpdateProgress(
-        Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction);
 void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction);
 
 FramesContainer *AppLayerFramesGetContainer(Flow *f);

diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c
index 6aeda8048f5836cf90d38d78f82670927885b82b..19624fd0d586d7cf3a3ec384e8c4ce126dcf20df 100644 (file)
--- a/src/detect-engine-frame.c
+++ b/src/detect-engine-frame.c
@@ -89,6 +89,7 @@ static void PrefilterMpmFrame(DetectEngineThreadCtx *det_ctx, const void *pectx,
             det_ctx, ctx->transforms, p, frames, frame, ctx->list_id, idx, true);
     if (buffer == NULL)
         return;
+    BUG_ON(buffer->orig_len > frame->len);
 
     const uint32_t data_len = buffer->inspect_len;
     const uint8_t *data = buffer->inspect;
@@ -192,6 +193,86 @@ static InspectionBuffer *DetectFrame2InspectBufferUdp(DetectEngineThreadCtx *det
     return buffer;
 }
 
+struct FrameStreamData {
+    DetectEngineThreadCtx *det_ctx;
+    const DetectEngineTransforms *transforms;
+    const Frame *frame;
+    int list_id;
+    uint32_t idx;
+    uint64_t frame_data_offset_abs;
+    uint64_t frame_start_offset_abs;
+};
+
+static int FrameStreamDataFunc(
+        void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
+{
+    struct FrameStreamData *fsd = cb_data;
+    SCLogDebug("fsd %p { det_ct:%p, transforms:%p, frame:%p, list_id:%d, idx:%u, "
+               "frame_data_offset_abs:%" PRIu64 ", frame_start_offset_abs:%" PRIu64
+               " }, input: %p, input_len:%u, offset:%" PRIu64,
+            fsd, fsd->det_ctx, fsd->transforms, fsd->frame, fsd->list_id, fsd->idx,
+            fsd->frame_data_offset_abs, fsd->frame_start_offset_abs, input, input_len, offset);
+
+    InspectionBuffer *buffer =
+            InspectionBufferMultipleForListGet(fsd->det_ctx, fsd->list_id, fsd->idx);
+    BUG_ON(buffer == NULL);
+    SCLogDebug("buffer %p", buffer);
+
+    const Frame *frame = fsd->frame;
+    SCLogDebug("frame rel_offset:%" PRIi64, frame->rel_offset);
+    const uint8_t *data = input;
+    uint8_t ci_flags = 0;
+    uint32_t data_len;
+    if (fsd->frame_start_offset_abs == offset) {
+        ci_flags |= DETECT_CI_FLAGS_START;
+        SCLogDebug("have frame data start");
+
+        if (frame->len >= 0) {
+            data_len = MIN(input_len, frame->len);
+        } else {
+            data_len = input_len;
+        }
+
+        if (data_len == frame->len) {
+            ci_flags |= DETECT_CI_FLAGS_END;
+            SCLogDebug("have frame data end");
+        }
+    } else {
+        BUG_ON(offset < fsd->frame_data_offset_abs);
+
+        uint64_t frame_delta = offset - fsd->frame_start_offset_abs;
+        uint64_t request_delta =
+                offset -
+                fsd->frame_data_offset_abs; // diff between what we requested and what we got
+        BUG_ON(request_delta > frame_delta);
+
+        if (frame->len >= 0) {
+            if (frame_delta >= (uint64_t)frame->len) {
+                SCLogDebug("data entirely past frame");
+                return 1;
+            }
+            uint32_t adjusted_frame_len = (uint32_t)((uint64_t)frame->len - frame_delta);
+            SCLogDebug("frame len after applying offset %" PRIu64 ": %u", frame_delta,
+                    adjusted_frame_len);
+
+            data_len = MIN(adjusted_frame_len, input_len);
+            SCLogDebug("usable data len for frame: %u", data_len);
+
+            if ((uint64_t)data_len + frame_delta == (uint64_t)frame->len) {
+                ci_flags |= DETECT_CI_FLAGS_END;
+                SCLogDebug("have frame data end");
+            }
+        } else {
+            data_len = input_len;
+        }
+    }
+    // PrintRawDataFp(stdout, data, data_len);
+    InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len);
+    buffer->inspect_offset = frame->rel_offset < 0 ? -1 * frame->rel_offset : 0; // TODO review/test
+    buffer->flags = ci_flags;
+    return 1; // for now only the first chunk
+}
+
 InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Packet *p, const Frames *frames,
         const Frame *frame, const int list_id, const uint32_t idx, const bool first)
@@ -244,21 +325,15 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
             avail: 50 (complete)
      */
 
-    SCLogDebug("frame %" PRIi64 ", len %" PRIi64, frame->id, frame->len);
-
-    uint32_t data_len = 0;
-    const uint8_t *data = NULL;
+    SCLogDebug("frame %" PRIi64 ", len %" PRIi64 ", rel_offset %" PRIi64, frame->id, frame->len,
+            frame->rel_offset);
 
     uint64_t offset = STREAM_BASE_OFFSET(stream);
-    if (frame->rel_offset > 0 || frames->progress_rel) {
-        uint64_t frame_offset = 0;
-        if (frame->rel_offset >= 0) {
-            frame_offset = MAX((uint64_t)frame->rel_offset, (uint64_t)frames->progress_rel);
-        } else {
-            frame_offset = (uint64_t)frames->progress_rel;
-        }
-        offset += frame_offset;
+    if (frame->rel_offset > 0) {
+        offset += (uint64_t)frame->rel_offset;
     }
+    const int64_t frame_start_abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
+    BUG_ON(frame_start_abs_offset < 0);
 
     const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p);
 
@@ -266,68 +341,14 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
     if (usable <= offset)
         return NULL;
 
-    // TODO GAP handling
-    if (StreamingBufferGetDataAtOffset(&stream->sb, &data, &data_len, offset) == 0) {
-        return NULL;
-    }
+    struct FrameStreamData fsd = { det_ctx, transforms, frame, list_id, idx, offset,
+        (uint64_t)frame_start_abs_offset };
+    StreamReassembleForFrame(ssn, stream, FrameStreamDataFunc, &fsd, offset, eof);
+    SCLogDebug("offset %" PRIu64, offset);
 
-    const uint64_t data_right_edge = offset + data_len;
-    if (data_right_edge > usable)
-        data_len = usable - offset;
-
-    const int64_t frame_start_abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
-    const uint64_t usable_right_edge = MIN(data_right_edge, usable);
-
-    bool have_end = false;
-
-    if (frame->len > 0) {
-        const int64_t frame_avail_data_abs = (int64_t)usable_right_edge;
-        const int64_t frame_end_abs_offset = frame_start_abs_offset + frame->len;
-        have_end = (int64_t)usable_right_edge >= frame_end_abs_offset;
-
-        SCLogDebug("frame_end_abs_offset %" PRIi64 ", usable_right_edge %" PRIu64,
-                frame_end_abs_offset, usable_right_edge);
-
-        const int64_t avail_from_frame = MIN(frame_end_abs_offset, frame_avail_data_abs) - offset;
-        if (avail_from_frame < (int64_t)data_len) {
-            SCLogDebug("adjusted data len from %u to %" PRIi64, data_len, avail_from_frame);
-            data_len = (uint32_t)avail_from_frame;
-        }
-    }
-    const bool have_start = frame_start_abs_offset == (int64_t)offset;
-
-    if (data == NULL || data_len == 0) {
-        return NULL;
-    }
-
-    // TODO use eof too?
-    SCLogDebug("stream->min_inspect_depth %u", stream->min_inspect_depth);
-    if (data_len < frame->len && data_len < stream->min_inspect_depth) {
-        if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED || ssn->state == TCP_CLOSED ||
-                stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
-            SCLogDebug("EOF use available data: %u bytes", data_len);
-        } else {
-            SCLogDebug("not enough data to inspect now: have %u, want %u", data_len,
-                    stream->min_inspect_depth);
-            return NULL;
-        }
-    }
-
-    const uint8_t ci_flags =
-            (have_start ? DETECT_CI_FLAGS_START : 0) | (have_end ? DETECT_CI_FLAGS_END : 0);
-    SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s rel_offset %" PRIi64
-               " type %u len %" PRIi64 " ci_flags %02x (start:%s, end:%s)",
-            p->pcap_cnt, frame, frame->id,
-            AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type),
-            frame->rel_offset, frame->type, frame->len, ci_flags,
-            (ci_flags & DETECT_CI_FLAGS_START) ? "true" : "false",
-            (ci_flags & DETECT_CI_FLAGS_END) ? "true" : "false");
-    // PrintRawDataFp(stdout, data, MIN(32,data_len));
-
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
-    buffer->inspect_offset = frame->rel_offset < 0 ? -1 * frame->rel_offset : 0; // TODO review/test
-    buffer->flags = ci_flags;
-    return buffer;
+    InspectionBuffer *ret = InspectionBufferMultipleForListGet(det_ctx, list_id, idx);
+    SCLogDebug("ret %p", ret);
+    return ret;
 }
 
 /**

diff --git a/src/output-json-frame.c b/src/output-json-frame.c
index e0caf1de03099e399600af0d96e8fa592662f3c2..00e438f00d4c7676f809757a51293c81fae2c043 100644 (file)
--- a/src/output-json-frame.c
+++ b/src/output-json-frame.c
@@ -330,9 +330,9 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
 
             int64_t abs_offset = (int64_t)frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
             int64_t win = STREAM_APP_PROGRESS(stream) - abs_offset;
-            SCLogDebug("abs_offset %" PRIi64 ", frame->rel_offset %" PRIi64
-                       ", frames->progress_rel %d win %" PRIi64,
-                    abs_offset, frame->rel_offset, frames->progress_rel, win);
+            //            SCLogDebug("abs_offset %" PRIi64 ", frame->rel_offset %" PRIi64
+            //                       ", frames->progress_rel %d win %" PRIi64,
+            //                    abs_offset, frame->rel_offset, frames->progress_rel, win);
 
             if (!eof && win < frame->len && win < 2500) {
                 SCLogDebug("frame id %" PRIi64 " len %" PRIi64 ", win %" PRIi64

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 31cfaf49f7da538561124bc9cc1ad155069d289f..5956e1e5d5b2d94a34eb170cb44c879a5dbcfa8f 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -1389,14 +1389,6 @@ void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t prog
         stream = &ssn->server;
     }
 
-    /* Record updates */
-    if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
-        if (progress > STREAM_APP_PROGRESS(stream)) {
-            AppLayerFramesUpdateProgress(p->flow, stream, progress,
-                    PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
-        }
-    }
-
     if (progress > STREAM_RAW_PROGRESS(stream)) {
         uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
         stream->raw_progress_rel += slide;
@@ -1745,8 +1737,13 @@ end:
 int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback,
         void *cb_data, const uint64_t offset, const bool eof)
 {
+    /* take app progress as the right edge of used data. */
+    const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
+    SCLogDebug("app_progress %" PRIu64, app_progress);
+
     uint64_t unused = 0;
-    return StreamReassembleRawDo(ssn, stream, Callback, cb_data, offset, &unused, eof, false);
+    return StreamReassembleRawDo(
+            ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false);
 }
 
 int StreamReassembleRaw(TcpSession *ssn, const Packet *p,