frames: implement generic <alproto>.stream frames

Add a hard coded <alproto>.stream option for all stream data for
a protocol.

Starts at stream offset 0 or at the point of a protocol upgrade
in case of STARTTLS or CONNECT.

diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
index 95683a5c703cb6cb68162eb18097154b956690c3..4c7cca8bae7303fd5aaddf529e8757ae99649bfd 100644 (file)
--- a/src/app-layer-frames.c
+++ b/src/app-layer-frames.c
@@ -33,9 +33,12 @@
 static void FrameDebug(const char *prefix, const Frames *frames, const Frame *frame)
 {
 #ifdef DEBUG
-    const char *type_name =
-            frames ? AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type)
-                   : "<unknown>";
+    const char *type_name = "unknown";
+    if (frame->type == FRAME_STREAM_TYPE) {
+        type_name = "stream";
+    } else if (frames != NULL) {
+        type_name = AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type);
+    }
     SCLogDebug("[%s] %p: frame: %p type %u/%s id %" PRIi64 " flags %02x rel_offset:%" PRIi64
                ", len:%" PRIi64 ", events:%u %u/%u/%u/%u",
             prefix, frames, frame, frame->type, type_name, frame->id, frame->flags,
@@ -46,14 +49,17 @@ static void FrameDebug(const char *prefix, const Frames *frames, const Frame *fr
 
 Frame *FrameGetById(Frames *frames, const int64_t id)
 {
+    SCLogDebug("frames %p cnt %u, looking for %" PRIi64, frames, frames->cnt, id);
     for (uint16_t i = 0; i < frames->cnt; i++) {
         if (i < FRAMES_STATIC_CNT) {
             Frame *frame = &frames->sframes[i];
+            FrameDebug("get_by_id(static)", frames, frame);
             if (frame->id == id)
                 return frame;
         } else {
             const uint16_t o = i - FRAMES_STATIC_CNT;
             Frame *frame = &frames->dframes[o];
+            FrameDebug("get_by_id(dynamic)", frames, frame);
             if (frame->id == id)
                 return frame;
         }
@@ -570,10 +576,10 @@ Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice
 
     const uint64_t frame_start_rel = frame_start - STREAM_BASE_OFFSET(stream);
 #ifdef DEBUG
-    SCLogDebug("flow %p direction %s frame offset %" PRIu64 " (abs %" PRIu64
+    SCLogDebug("flow %p direction %s frame type %u offset %" PRIu64 " (abs %" PRIu64
                ") starting at %" PRIu64 " len %" PRIi64 " (offset %" PRIu64 ")",
-            f, dir == 0 ? "toserver" : "toclient", frame_start_rel, frame_start, frame_start, len,
-            stream_slice->offset);
+            f, dir == 0 ? "toserver" : "toclient", frame_type, frame_start_rel, frame_start,
+            frame_start, len, stream_slice->offset);
 #endif
     Frame *r = FrameNew(frames, (uint32_t)frame_start_rel, len);
     if (r != NULL) {
@@ -639,6 +645,8 @@ void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t
 Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
 {
     FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    SCLogDebug("get frame_id %" PRIi64 " direction %u/%s frames_container %p", frame_id, dir,
+            dir == 0 ? "toserver" : "toclient", frames_container);
     if (frames_container == NULL)
         return NULL;
 
@@ -648,6 +656,7 @@ Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
     } else {
         frames = &frames_container->toclient;
     }
+    SCLogDebug("frames %p", frames);
     return FrameGetById(frames, frame_id);
 }
 

diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
index 23cba1753a74300e94f5cada0d0cf81ae2f00ae9..c163298bf3eb820bc4de7fc30057d85ce4250424 100644 (file)
--- a/src/app-layer-frames.h
+++ b/src/app-layer-frames.h
@@ -26,6 +26,10 @@
 
 #include "rust.h"
 
+#define FRAME_STREAM_TYPE 255
+/** always the first frame to be created. TODO but what about protocol upgrades? */
+#define FRAME_STREAM_ID 1
+
 typedef int64_t FrameId;
 
 enum {

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 12173e74e5216e3b851eeaa3aa80624187b6275b..c357a7ddc198643bf294ca5eb713d38388d37d21 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -1254,6 +1254,57 @@ static inline void SetEOFFlags(AppLayerParserState *pstate, const uint8_t flags)
     }
 }
 
+/** \internal
+ *  \brief create/close stream frames
+ *  On first invocation of TCP parser in a direction, create a <alproto>.stream frame.
+ *  On STREAM_EOF, set the final length. */
+static void HandleStreamFrames(Flow *f, StreamSlice stream_slice, const uint8_t *input,
+        const uint32_t input_len, const uint8_t flags)
+{
+    const uint8_t direction = (flags & STREAM_TOSERVER) ? 0 : 1;
+    AppLayerParserState *pstate = f->alparser;
+
+    /* setup the generic stream frame */
+    if (((direction == 0 && (pstate->flags & APP_LAYER_PARSER_SFRAME_TS) == 0) ||
+                (direction == 1 && (pstate->flags & APP_LAYER_PARSER_SFRAME_TC) == 0)) &&
+            input != NULL && f->proto == IPPROTO_TCP) {
+        Frame *frame = AppLayerFrameGetById(f, direction, FRAME_STREAM_ID);
+        if (frame == NULL) {
+            int64_t frame_len = -1;
+            if (flags & STREAM_EOF)
+                frame_len = input_len;
+
+            frame = AppLayerFrameNewByAbsoluteOffset(
+                    f, &stream_slice, stream_slice.offset, frame_len, direction, FRAME_STREAM_TYPE);
+            if (frame) {
+                SCLogDebug("opened: frame %p id %" PRIi64, frame, frame->id);
+                frame->flags = FRAME_FLAG_ENDS_AT_EOF; // TODO logic is not yet implemented
+                DEBUG_VALIDATE_BUG_ON(
+                        frame->id != 1); // should always be the first frame that is created
+            }
+            if (direction == 0) {
+                pstate->flags |= APP_LAYER_PARSER_SFRAME_TS;
+            } else {
+                pstate->flags |= APP_LAYER_PARSER_SFRAME_TC;
+            }
+        }
+    } else if (flags & STREAM_EOF) {
+        Frame *frame = AppLayerFrameGetById(f, direction, FRAME_STREAM_ID);
+        SCLogDebug("EOF closing: frame %p", frame);
+        if (frame) {
+            /* calculate final frame length */
+            const TcpSession *ssn = f->protoctx;
+            const TcpStream *stream = (direction == 0) ? &ssn->client : &ssn->server;
+            int64_t slice_o =
+                    (int64_t)stream_slice.offset - (int64_t)stream->sb.region.stream_offset;
+            int64_t frame_len = slice_o + ((int64_t)-1 * frame->rel_offset) + (int64_t)input_len;
+            SCLogDebug("%s: EOF frame->rel_offset %" PRIi64 " -> %" PRIi64 ": o %" PRIi64,
+                    AppProtoToString(f->alproto), frame->rel_offset, frame_len, slice_o);
+            frame->len = frame_len;
+        }
+    }
+}
+
 static void Setup(Flow *f, const uint8_t direction, const uint8_t *input, uint32_t input_len,
         const uint8_t flags, StreamSlice *as)
 {
@@ -1349,6 +1400,8 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow
     if (input_len > 0 || (flags & STREAM_EOF)) {
         Setup(f, flags & (STREAM_TOSERVER | STREAM_TOCLIENT), input, input_len, flags,
                 &stream_slice);
+        HandleStreamFrames(f, stream_slice, input, input_len, flags);
+
 #ifdef DEBUG
         if (((stream_slice.flags & STREAM_TOSERVER) &&
                     stream_slice.offset >= g_eps_applayer_error_offset_ts)) {

diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index 902a1a949ae8bad35d268429d09a8d128b3169df..ead069184c91745f567242c6b34300000c62f7ae 100644 (file)
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -40,6 +40,8 @@
 #define APP_LAYER_PARSER_EOF_TC                BIT_U16(6)
 #define APP_LAYER_PARSER_TRUNC_TS              BIT_U16(7)
 #define APP_LAYER_PARSER_TRUNC_TC              BIT_U16(8)
+#define APP_LAYER_PARSER_SFRAME_TS             BIT_U16(9)
+#define APP_LAYER_PARSER_SFRAME_TC             BIT_U16(10)
 
 /* Flags for AppLayerParserProtoCtx. */
 #define APP_LAYER_PARSER_OPT_ACCEPT_GAPS        BIT_U32(0)

diff --git a/src/detect-frame.c b/src/detect-frame.c
index 584636993e2097cbd47c6de97f836d58b47f0abc..8df5df9366f2d9e17dcbd3b46ddcedb898fdf509 100644 (file)
--- a/src/detect-frame.c
+++ b/src/detect-frame.c
@@ -24,6 +24,7 @@
 #include "decode.h"
 #include "detect.h"
 
+#include "app-layer-frames.h"
 #include "app-layer-parser.h"
 
 #include "detect-parse.h"
@@ -106,15 +107,21 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s
 
     const char *frame_str = is_short ? str : val;
     int raw_frame_type = -1;
-    if (is_tcp)
-        raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, keyword_alproto, frame_str);
+    if (is_tcp) {
+        if (strcmp(frame_str, "stream") == 0) {
+            raw_frame_type = FRAME_STREAM_TYPE;
+        } else {
+            raw_frame_type =
+                    AppLayerParserGetFrameIdByName(IPPROTO_TCP, keyword_alproto, frame_str);
+        }
+    }
     if (is_udp && raw_frame_type < 0)
         raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, keyword_alproto, frame_str);
     if (raw_frame_type < 0) {
         SCLogError("unknown frame '%s' for protocol '%s'", frame_str, proto);
         return -1;
     }
-    BUG_ON(raw_frame_type >= UINT8_MAX);
+    BUG_ON(raw_frame_type > UINT8_MAX);
 
     if (is_short) {
         snprintf(buffer_name, sizeof(buffer_name), "%s.%s", AppProtoToString(s->alproto), str);