From: Victor Julien <vjulien@oisf.net>
Date: Fri, 3 Dec 2021 06:40:56 +0000 (+0100)
Subject: app/frames: initial support
X-Git-Tag: suricata-7.0.0-beta1~1047
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1556e86c7db139057e8ad0a964dc77d010b8134b;p=thirdparty%2Fsuricata.git

app/frames: initial support

The idea of stream frames is that the applayer parsers can tag PDUs and
other arbitrary frames in the stream while parsing. These frames can then
be inspected from the rule language. This will allow rules that are more
precise and less costly.

The frames are stored per direction in the `AppLayerParserState` and will only
be initialized when actual frames are in use. The per direction storage has a
fixed size static portion and dynamic support for a larger number. This is done
for effeciency.

When the Stream Buffer slides, frames are updated as they use offsets relative
to the stream. A negative offset is used for frames that started before the
current window.

Frames have events to inspect/log parser errors that don't fit the TX model.

Frame id starts at 1. So implementations can keep track of frame ids where 0
is not set.

Frames affect TCP window sliding. The frames keep a "left edge" which
signifies how much data to keep for frames that are still in progress.
---

diff --git a/src/Makefile.am b/src/Makefile.am
index f7edaaa518..61fdfae5e8 100755
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -22,6 +22,7 @@ noinst_HEADERS = \
 	app-layer-enip.h \
 	app-layer-events.h \
 	app-layer-expectation.h \
+	app-layer-frames.h \
 	app-layer-ftp.h \
 	app-layer.h \
 	app-layer-htp-body.h \
@@ -611,6 +612,7 @@ libsuricata_c_a_SOURCES = \
 	app-layer-events.c \
 	app-layer-expectation.c \
 	app-layer-ftp.c \
+	app-layer-frames.c \
 	app-layer-htp-body.c \
 	app-layer-htp.c \
 	app-layer-htp-file.c \
diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c
new file mode 100644
index 0000000000..87c0fdac50
--- /dev/null
+++ b/src/app-layer-frames.c
@@ -0,0 +1,781 @@
+/* Copyright (C) 2007-2021 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have frameeived a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ */
+
+#include "suricata-common.h"
+#include "debug.h"
+#include "util-print.h"
+
+#include "stream-tcp.h"
+#include "app-layer-frames.h"
+
+static void FrameDebug(const char *prefix, const Frames *frames, const Frame *frame)
+{
+#ifdef DEBUG
+    const char *type_name =
+            frames ? AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type)
+                   : "<unknown>";
+    SCLogDebug("[%s] %p: frame: %p type %u/%s id %" PRIi64 " flags %02x rel_offset:%" PRIi64
+               ", len:%" PRIi64 ", events:%u %u/%u/%u/%u",
+            prefix, frames, frame, frame->type, type_name, frame->id, frame->flags,
+            frame->rel_offset, frame->len, frame->event_cnt, frame->events[0], frame->events[1],
+            frame->events[2], frame->events[3]);
+#endif
+}
+
+Frame *FrameGetById(Frames *frames, const int64_t id)
+{
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            Frame *frame = &frames->sframes[i];
+            if (frame->id == id)
+                return frame;
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            Frame *frame = &frames->dframes[o];
+            if (frame->id == id)
+                return frame;
+        }
+    }
+    return NULL;
+}
+
+Frame *FrameGetByIndex(Frames *frames, const uint32_t idx)
+{
+    if (idx >= frames->cnt)
+        return NULL;
+
+    if (idx < FRAMES_STATIC_CNT) {
+        Frame *frame = &frames->sframes[idx];
+        FrameDebug("get_by_idx(s)", frames, frame);
+        return frame;
+    } else {
+        const uint16_t o = idx - FRAMES_STATIC_CNT;
+        Frame *frame = &frames->dframes[o];
+        FrameDebug("get_by_idx(d)", frames, frame);
+        return frame;
+    }
+}
+
+// TODO review rel_offset logic. App-layer passes STREAM_APP_PROGRESS as
+// offset, but I think we're using rel_offset relative to BASE_PROGRESS
+// here which changes only on slide.
+static Frame *FrameNew(Frames *frames, int64_t rel_offset, int64_t len)
+{
+    BUG_ON(frames == NULL);
+
+    if (frames->cnt < FRAMES_STATIC_CNT) {
+        Frame *frame = &frames->sframes[frames->cnt];
+        frames->sframes[frames->cnt].rel_offset = rel_offset;
+        frames->sframes[frames->cnt].len = len;
+        frames->sframes[frames->cnt].id = ++frames->base_id;
+        frames->cnt++;
+        return frame;
+    } else if (frames->dframes == NULL) {
+        BUG_ON(frames->dyn_size != 0);
+        BUG_ON(frames->cnt != FRAMES_STATIC_CNT);
+
+        frames->dframes = SCCalloc(8, sizeof(Frame));
+        if (frames->dframes == NULL) {
+            return NULL;
+        }
+        frames->cnt++;
+        BUG_ON(frames->cnt != FRAMES_STATIC_CNT + 1);
+
+        frames->dyn_size = 8;
+        frames->dframes[0].rel_offset = rel_offset;
+        frames->dframes[0].len = len;
+        frames->dframes[0].id = ++frames->base_id;
+        return &frames->dframes[0];
+    } else {
+        BUG_ON(frames->cnt < FRAMES_STATIC_CNT);
+
+        /* need to handle dynamic storage of frames now */
+        const uint16_t dyn_cnt = frames->cnt - FRAMES_STATIC_CNT;
+        if (dyn_cnt < frames->dyn_size) {
+            BUG_ON(frames->dframes == NULL);
+
+            // fall through
+        } else {
+            if (frames->dyn_size == 256) {
+                SCLogDebug("limit reached! 256 dynamic frames already");
+                // limit reached
+                // TODO figure out if this should lead to an event of sorts
+                return NULL;
+            }
+
+            /* realloc time */
+            uint16_t new_dyn_size = frames->dyn_size * 2;
+            uint32_t new_alloc_size = new_dyn_size * sizeof(Frame);
+
+            void *ptr = SCRealloc(frames->dframes, new_alloc_size);
+            if (ptr == NULL) {
+                return NULL;
+            }
+
+            memset((uint8_t *)ptr + (frames->dyn_size * sizeof(Frame)), 0x00,
+                    (frames->dyn_size * sizeof(Frame)));
+            frames->dframes = ptr;
+            frames->dyn_size = new_dyn_size;
+        }
+
+        frames->cnt++;
+        frames->dframes[dyn_cnt].rel_offset = rel_offset;
+        frames->dframes[dyn_cnt].len = len;
+        frames->dframes[dyn_cnt].id = ++frames->base_id;
+        return &frames->dframes[dyn_cnt];
+    }
+}
+
+static void FrameClean(Frame *frame)
+{
+    memset(frame, 0, sizeof(*frame));
+}
+
+static void FrameCopy(Frame *dst, Frame *src)
+{
+    memcpy(dst, src, sizeof(*dst));
+}
+
+static void AppLayerFrameDumpForFrames(const char *prefix, const Frames *frames)
+{
+    SCLogDebug("prefix: %s", prefix);
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            const Frame *frame = &frames->sframes[i];
+            FrameDebug(prefix, frames, frame);
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            const Frame *frame = &frames->dframes[o];
+            FrameDebug(prefix, frames, frame);
+        }
+    }
+    SCLogDebug("prefix: %s", prefix);
+}
+
+static inline uint64_t FrameLeftEdge(
+        const TcpStream *stream, const Frame *frame, const int64_t base_offset)
+{
+    const int64_t app_progress = STREAM_APP_PROGRESS(stream);
+    BUG_ON(base_offset > app_progress);
+
+    const int64_t frame_offset = base_offset + frame->rel_offset;
+    const int64_t frame_data = app_progress - frame_offset;
+
+    SCLogDebug("base_offset %" PRIi64 ", app_progress %" PRIi64, base_offset, app_progress);
+    SCLogDebug("frame_offset %" PRIi64 ", frame_data %" PRIi64 ", frame->len %" PRIi64,
+            frame_offset, frame_data, frame->len);
+    BUG_ON(frame_offset < 0);
+    BUG_ON(frame_offset > app_progress);
+
+    /* length unknown, make sure to have at least 2500 */
+    if (frame->len < 0) {
+        if (frame_data <= 2500) {
+            SCLogDebug("got <= 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
+                    frame_offset);
+            return frame_offset;
+        } else {
+            SCLogDebug("got > 2500 bytes (%" PRIu64 "), returning offset %" PRIu64, frame_data,
+                    (frame_offset + (frame_data - 2500)));
+            return frame_offset + (frame_data - 2500);
+        }
+
+        /* length specified */
+    } else {
+        /* have all data for the frame, we can skip it */
+        if (frame->len <= frame_data) {
+            uint64_t x = frame_offset + frame_data;
+            SCLogDebug("x %" PRIu64, x);
+            return x;
+            /*
+
+                [ stream      <frame_data> ]
+                             [ frame        .......]
+
+             */
+        } else if (frame_data < 2500) {
+            uint64_t x = frame_offset;
+            SCLogDebug("x %" PRIu64, x);
+            return x;
+        } else {
+            uint64_t x = frame_offset + (frame_data - 2500);
+            SCLogDebug("x %" PRIu64, x);
+            return x;
+        }
+    }
+}
+#if 0
+static inline uint64_t FramesLeftEdge(const TcpStream *stream, const Frames *frames)
+{
+    uint64_t le = STREAM_APP_PROGRESS(stream);
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            const Frame *frame = &frames->sframes[i];
+            le = MIN(le, FrameLeftEdge(stream, frame));
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            const Frame *frame = &frames->dframes[o];
+            le = MIN(le, FrameLeftEdge(stream, frame));
+        }
+    }
+    return le;
+}
+#endif
+
+/** Stream buffer slides forward, we need to update and age out
+ *  frame offsets/frames. Aging out means we move existing frames
+ *  into the slots we'd free up.
+ *
+ *  Start:
+ *
+ *  [ stream ]
+ *    [ frame   ...........]
+ *      rel_offset: 2
+ *      len: 19
+ *
+ *  Slide:
+ *         [ stream ]
+ *    [ frame ....          .]
+ *      rel_offset: -10
+ *       len: 19
+ *
+ *  Slide:
+ *                [ stream ]
+ *    [ frame ...........    ]
+ *      rel_offset: -16
+ *      len: 19
+ */
+static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, const uint32_t slide)
+{
+    SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
+               ", next %" PRIu64,
+            (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
+            STREAM_BASE_OFFSET(stream), STREAM_BASE_OFFSET(stream) + slide);
+    BUG_ON(frames == NULL);
+    SCLogDebug("%s frames %p: sliding %u bytes", ds, frames, slide);
+    uint64_t le = STREAM_APP_PROGRESS(stream);
+
+    if (slide >= frames->progress_rel)
+        frames->progress_rel = 0;
+    else
+        frames->progress_rel -= slide;
+
+    const uint64_t next_base = STREAM_BASE_OFFSET(stream) + slide;
+    const uint16_t start = frames->cnt;
+    uint16_t removed = 0;
+    uint16_t x = 0;
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            Frame *frame = &frames->sframes[i];
+            FrameDebug("slide(s)", frames, frame);
+            if (frame->len >= 0 &&
+                    frame->rel_offset + frame->len <= (int64_t)slide) { // TODO check seems off
+                // remove by not incrementing 'x'
+                SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
+                FrameClean(frame);
+                removed++;
+            } else {
+                Frame *nframe = &frames->sframes[x];
+                FrameCopy(nframe, frame);
+                nframe->rel_offset -= slide; /* turns negative if start if before window */
+                if (frame != nframe) {
+                    FrameClean(frame);
+                }
+                le = MIN(le, FrameLeftEdge(stream, nframe, next_base));
+                x++;
+            }
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            Frame *frame = &frames->dframes[o];
+            FrameDebug("slide(d)", frames, frame);
+            if (frame->len >= 0 && frame->rel_offset + frame->len <= (int64_t)slide) {
+                // remove by not incrementing 'x'
+                SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
+                FrameClean(frame);
+                removed++;
+            } else {
+                Frame *nframe;
+                if (x >= FRAMES_STATIC_CNT) {
+                    nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
+                } else {
+                    nframe = &frames->sframes[x];
+                }
+                FrameCopy(nframe, frame);
+                nframe->rel_offset -= slide; /* turns negative if start is before window */
+                if (frame != nframe) {
+                    FrameClean(frame);
+                }
+                le = MIN(le, FrameLeftEdge(stream, nframe, next_base));
+                x++;
+            }
+        }
+    }
+    frames->cnt = x;
+    uint64_t o = STREAM_BASE_OFFSET(stream) + slide;
+    frames->left_edge_rel = le - (STREAM_BASE_OFFSET(stream) + slide);
+
+#ifdef DEBUG
+    SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
+               " (+slide), cnt %u, removed %u, start %u",
+            (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream) + slide,
+            frames->left_edge_rel, STREAM_BASE_OFFSET(stream) + slide, frames->cnt, removed, start);
+    char pf[32] = "";
+    snprintf(pf, sizeof(pf), "%s:post_slide", ds);
+    AppLayerFrameDumpForFrames(pf, frames);
+#endif
+    BUG_ON(o > le);
+    BUG_ON(x != start - removed);
+    return 0;
+}
+
+void AppLayerFramesUpdateProgress(
+        Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction)
+{
+    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    if (frames_container == NULL)
+        return;
+
+    Frames *frames;
+    if (direction == STREAM_TOSERVER) {
+        frames = &frames_container->toserver;
+    } else {
+        frames = &frames_container->toclient;
+    }
+
+    const uint32_t slide = progress - STREAM_APP_PROGRESS(stream);
+    frames->progress_rel += slide;
+}
+
+void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction)
+{
+    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    if (frames_container == NULL)
+        return;
+    Frames *frames;
+    TcpSession *ssn = f->protoctx;
+    TcpStream *stream;
+    if (direction == STREAM_TOSERVER) {
+        stream = &ssn->client;
+        frames = &frames_container->toserver;
+        FrameSlide("toserver", frames, stream, slide);
+    } else {
+        stream = &ssn->server;
+        frames = &frames_container->toclient;
+        FrameSlide("toclient", frames, stream, slide);
+    }
+}
+
+static void FrameFreeSingleFrame(Frames *frames, Frame *r)
+{
+    FrameDebug("free", frames, r);
+    FrameClean(r);
+}
+
+void FramesFree(Frames *frames)
+{
+    BUG_ON(frames == NULL);
+
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            Frame *r = &frames->sframes[i];
+            FrameFreeSingleFrame(frames, r);
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            Frame *r = &frames->dframes[o];
+            FrameFreeSingleFrame(frames, r);
+        }
+    }
+    SCFree(frames->dframes);
+    frames->dframes = NULL;
+}
+
+/** \brief create new frame using a pointer to start of the frame
+ */
+Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
+        const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
+{
+    SCLogDebug("stream_slice offset %" PRIu64, stream_slice->offset);
+    SCLogDebug("frame_start %p stream_slice->input %p", frame_start, stream_slice->input);
+
+    /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
+#if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
+    if (f->protoctx == NULL)
+        return NULL;
+    if (frame_start < stream_slice->input ||
+            frame_start >= stream_slice->input + stream_slice->input_len)
+        return NULL;
+#endif
+    BUG_ON(frame_start < stream_slice->input);
+    BUG_ON(stream_slice->input == NULL);
+    BUG_ON(f->proto != IPPROTO_TCP);
+    BUG_ON(f->protoctx == NULL);
+
+    ptrdiff_t ptr_offset = frame_start - stream_slice->input;
+#ifdef DEBUG
+    uint64_t offset = ptr_offset + stream_slice->offset;
+    SCLogDebug("flow %p direction %s frame %p starting at %" PRIu64 " len %" PRIi64
+               " (offset %" PRIu64 ")",
+            f, dir == 0 ? "toserver" : "toclient", frame_start, offset, len, stream_slice->offset);
+#endif
+    BUG_ON(f->alparser == NULL);
+
+    FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
+    if (frames_container == NULL)
+        return NULL;
+
+    TcpStream *stream;
+    TcpSession *ssn = f->protoctx;
+    Frames *frames;
+    if (dir == 0) {
+        frames = &frames_container->toserver;
+        stream = &ssn->client;
+    } else {
+        frames = &frames_container->toclient;
+        stream = &ssn->server;
+    }
+
+    int64_t abs_frame_offset = stream_slice->offset + (int64_t)ptr_offset;
+    int64_t rel_offset = abs_frame_offset - STREAM_BASE_OFFSET(stream);
+
+    Frame *r = FrameNew(frames, rel_offset, len);
+    if (r != NULL) {
+        r->type = frame_type;
+    }
+    return r;
+}
+
+/** \brief create new frame using a relative offset from the start of the stream slice
+ */
+Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice,
+        const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
+{
+    /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
+#if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
+    if (f->protoctx == NULL)
+        return NULL;
+    if (stream_slice->input == NULL)
+        return NULL;
+#endif
+    BUG_ON(stream_slice->input == NULL);
+    BUG_ON(f->proto != IPPROTO_TCP);
+    BUG_ON(f->protoctx == NULL);
+    BUG_ON(f->alparser == NULL);
+
+    FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
+    if (frames_container == NULL)
+        return NULL;
+
+    TcpStream *stream;
+    TcpSession *ssn = f->protoctx;
+    Frames *frames;
+    if (dir == 0) {
+        frames = &frames_container->toserver;
+        stream = &ssn->client;
+    } else {
+        frames = &frames_container->toclient;
+        stream = &ssn->server;
+    }
+
+    const uint64_t base = STREAM_BASE_OFFSET(stream);
+#ifdef DEBUG
+    const uint64_t app = STREAM_APP_PROGRESS(stream);
+    const uint64_t app_offset = app - base;
+    const uint64_t slice_offset = stream_slice->offset - base;
+
+    SCLogDebug("app %" PRIu64 ", base %" PRIu64 ", slice %" PRIu64, app, base, slice_offset);
+    SCLogDebug("app_offset %" PRIu64 ", slice_offset %" PRIu64, app_offset, slice_offset);
+#endif
+    const uint64_t frame_abs_offset = (uint64_t)frame_start_rel + stream_slice->offset;
+    const uint64_t frame_base_offset = frame_abs_offset - base;
+
+    SCLogDebug("frame_start_rel %u frame_abs_offset %" PRIu64 ", frame_base_offset %" PRIu64,
+            frame_start_rel, frame_abs_offset, frame_base_offset);
+
+    int64_t rel_offset = frame_base_offset;
+#ifdef DEBUG
+    const char *type_name = AppLayerParserGetFrameNameById(f->proto, f->alproto, frame_type);
+    SCLogDebug("flow %p direction %s frame offset %u rel_offset %" PRIi64 " (abs %" PRIu64
+               ") starting at %" PRIu64 " len %" PRIi64 " (offset %" PRIu64 ") type %u/%s",
+            f, dir == 0 ? "toserver" : "toclient", frame_start_rel, rel_offset, frame_abs_offset,
+            frame_abs_offset, len, stream_slice->offset, frame_type, type_name);
+#endif
+
+    Frame *r = FrameNew(frames, rel_offset, len);
+    if (r != NULL) {
+        r->type = frame_type;
+    }
+    return r;
+}
+
+void AppLayerFrameDump(Flow *f)
+{
+    if (f->proto == IPPROTO_TCP && f->protoctx && f->alparser) {
+        FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+        if (frames_container != NULL) {
+            AppLayerFrameDumpForFrames("toserver::dump", &frames_container->toserver);
+            AppLayerFrameDumpForFrames("toclient::dump", &frames_container->toclient);
+        }
+    }
+}
+
+/** \brief create new frame using the absolute offset from the start of the stream
+ */
+Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice,
+        const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type)
+{
+    /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
+#if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
+    if (f->protoctx == NULL)
+        return NULL;
+    if (stream_slice->input == NULL)
+        return NULL;
+#endif
+    BUG_ON(stream_slice->input == NULL);
+    BUG_ON(f->proto != IPPROTO_TCP);
+    BUG_ON(f->protoctx == NULL);
+    BUG_ON(f->alparser == NULL);
+    BUG_ON(frame_start < stream_slice->offset);
+    BUG_ON(frame_start - stream_slice->offset >= (uint64_t)INT_MAX);
+
+    FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
+    if (frames_container == NULL)
+        return NULL;
+
+    TcpSession *ssn = f->protoctx;
+    TcpStream *stream;
+    Frames *frames;
+    if (dir == 0) {
+        stream = &ssn->client;
+        frames = &frames_container->toserver;
+    } else {
+        stream = &ssn->server;
+        frames = &frames_container->toclient;
+    }
+
+    const uint64_t frame_start_rel = frame_start - STREAM_BASE_OFFSET(stream);
+#ifdef DEBUG
+    SCLogDebug("flow %p direction %s frame offset %" PRIu64 " (abs %" PRIu64
+               ") starting at %" PRIu64 " len %" PRIi64 " (offset %" PRIu64 ")",
+            f, dir == 0 ? "toserver" : "toclient", frame_start_rel, frame_start, frame_start, len,
+            stream_slice->offset);
+#endif
+    Frame *r = FrameNew(frames, (uint32_t)frame_start_rel, len);
+    if (r != NULL) {
+        r->type = frame_type;
+    }
+    return r;
+}
+
+void AppLayerFrameAddEvent(Frame *r, uint8_t e)
+{
+    if (r != NULL) {
+        if (r->event_cnt < 4) { // TODO
+            r->events[r->event_cnt++] = e;
+        }
+        FrameDebug("add_event", NULL, r);
+    }
+}
+
+void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e)
+{
+    Frame *frame = AppLayerFrameGetById(f, dir, id);
+    AppLayerFrameAddEvent(frame, e);
+}
+
+FrameId AppLayerFrameGetId(Frame *r)
+{
+    if (r != NULL) {
+        return r->id;
+    } else {
+        return -1;
+    }
+}
+
+void AppLayerFrameSetLength(Frame *frame, int64_t len)
+{
+    if (frame != NULL) {
+        frame->len = len;
+        FrameDebug("set_length", NULL, frame);
+    }
+}
+
+void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len)
+{
+    Frame *frame = AppLayerFrameGetById(f, dir, id);
+    AppLayerFrameSetLength(frame, len);
+}
+
+void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id)
+{
+    if (r != NULL) {
+        r->flags |= FRAME_FLAG_TX_ID_SET;
+        r->tx_id = tx_id;
+        FrameDebug("set_txid", NULL, r);
+    }
+}
+
+void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id)
+{
+    Frame *frame = AppLayerFrameGetById(f, dir, id);
+    AppLayerFrameSetTxId(frame, tx_id);
+}
+
+Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id)
+{
+    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    if (frames_container == NULL)
+        return NULL;
+
+    Frames *frames;
+    if (dir == 0) {
+        frames = &frames_container->toserver;
+    } else {
+        frames = &frames_container->toclient;
+    }
+    return FrameGetById(frames, frame_id);
+}
+
+static inline bool FrameIsDone(
+        const Frame *frame, const uint64_t abs_offset, const uint64_t abs_right_edge)
+{
+    /* frame with negative length means we don't know the size yet. */
+    if (frame->len < 0)
+        return false;
+
+    const int64_t frame_abs_offset = (int64_t)abs_offset + frame->rel_offset;
+    const int64_t frame_right_edge = frame_abs_offset + frame->len;
+    if ((uint64_t)frame_right_edge <= abs_right_edge) {
+        SCLogDebug("frame %p id %" PRIi64 " is done", frame, frame->id);
+        return true;
+    }
+    return false;
+}
+
+static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
+{
+    const uint64_t frames_le_start = (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream);
+    SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64,
+            (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
+            STREAM_BASE_OFFSET(stream));
+    const uint64_t abs_offset = STREAM_BASE_OFFSET(stream) + (uint64_t)frames->progress_rel;
+    const uint64_t acked = StreamTcpGetUsable(stream, eof);
+    uint64_t le = STREAM_APP_PROGRESS(stream);
+
+    const uint16_t start = frames->cnt;
+    uint16_t removed = 0;
+    uint16_t x = 0;
+    for (uint16_t i = 0; i < frames->cnt; i++) {
+        if (i < FRAMES_STATIC_CNT) {
+            Frame *frame = &frames->sframes[i];
+            FrameDebug("prune(s)", frames, frame);
+            if (eof || FrameIsDone(frame, abs_offset, acked)) {
+                // remove by not incrementing 'x'
+                SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
+                FrameDebug("remove(s)", frames, frame);
+                FrameClean(frame);
+                removed++;
+            } else {
+                const uint64_t fle = FrameLeftEdge(stream, frame, STREAM_BASE_OFFSET(stream));
+                le = MIN(le, fle);
+                SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
+                Frame *nframe = &frames->sframes[x];
+                FrameCopy(nframe, frame);
+                if (frame != nframe) {
+                    FrameClean(frame);
+                }
+                x++;
+            }
+        } else {
+            const uint16_t o = i - FRAMES_STATIC_CNT;
+            Frame *frame = &frames->dframes[o];
+            FrameDebug("prune(d)", frames, frame);
+            if (eof || FrameIsDone(frame, abs_offset, acked)) {
+                // remove by not incrementing 'x'
+                SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
+                FrameDebug("remove(d)", frames, frame);
+                FrameClean(frame);
+                removed++;
+            } else {
+                const uint64_t fle = FrameLeftEdge(stream, frame, STREAM_BASE_OFFSET(stream));
+                le = MIN(le, fle);
+                SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
+                Frame *nframe;
+                if (x >= FRAMES_STATIC_CNT) {
+                    nframe = &frames->dframes[x - FRAMES_STATIC_CNT];
+                } else {
+                    nframe = &frames->sframes[x];
+                }
+                FrameCopy(nframe, frame);
+                if (frame != nframe) {
+                    FrameClean(frame);
+                }
+                x++;
+            }
+        }
+    }
+    frames->cnt = x;
+    frames->left_edge_rel = le - STREAM_BASE_OFFSET(stream);
+#ifdef DEBUG
+    SCLogDebug("end: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64
+               ", cnt %u, removed %u, start %u",
+            (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel,
+            STREAM_BASE_OFFSET(stream), frames->cnt, removed, start);
+    AppLayerFrameDumpForFrames("post_slide", frames);
+#endif
+    BUG_ON(le < STREAM_BASE_OFFSET(stream));
+    if (frames->cnt > 0) { // if we removed all this can fail
+        BUG_ON(frames_le_start > le);
+    }
+    BUG_ON(x != start - removed);
+}
+
+void FramesPrune(Flow *f, Packet *p)
+{
+    if (f->protoctx == NULL)
+        return;
+    FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    if (frames_container == NULL)
+        return;
+
+    Frames *frames;
+    TcpSession *ssn = f->protoctx;
+
+    if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
+        AppLayerFramesFreeContainer(f);
+        return;
+    }
+
+    TcpStream *stream;
+    if (PKT_IS_TOSERVER(p)) {
+        stream = &ssn->client;
+        frames = &frames_container->toserver;
+    } else {
+        stream = &ssn->server;
+        frames = &frames_container->toclient;
+    }
+
+    const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p) ||
+                     (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
+    SCLogDebug("eof %s", eof ? "TRUE" : "false");
+    FramePrune(frames, stream, eof);
+}
diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h
new file mode 100644
index 0000000000..dce2cc36a4
--- /dev/null
+++ b/src/app-layer-frames.h
@@ -0,0 +1,110 @@
+/* Copyright (C) 2021 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have frameeived a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __APP_LAYER_FRAMES_H__
+#define __APP_LAYER_FRAMES_H__
+
+#include "app-layer-events.h"
+#include "detect-engine-state.h"
+#include "util-file.h"
+#include "stream-tcp-private.h"
+#include "rust.h"
+#include "app-layer-parser.h"
+
+typedef int64_t FrameId;
+
+enum {
+    FRAME_FLAGE_TX_ID_SET,
+#define FRAME_FLAG_TX_ID_SET BIT_U8(FRAME_FLAGE_TX_ID_SET)
+    FRAME_FLAGE_ENDS_AT_EOF,
+#define FRAME_FLAG_ENDS_AT_EOF BIT_U8(FRAME_FLAGE_ENDS_AT_EOF)
+};
+
+typedef struct Frame {
+    uint8_t type;  /**< protocol specific field type. E.g. NBSS.HDR or SMB.DATA */
+    uint8_t flags; /**< frame flags: FRAME_FLAG_* */
+    uint8_t event_cnt;
+    // TODO one event per frame enough?
+    uint8_t events[4];  /**< per frame store for events */
+    int64_t rel_offset; /**< relative offset in the stream on top of Stream::stream_offset (if
+                           negative the start if before the stream data) */
+    int64_t len;
+    int64_t id;
+    uint64_t tx_id; /**< tx_id to match this frame. UINT64T_MAX if not used. */
+} Frame;
+// size 40
+
+#define FRAMES_STATIC_CNT 3
+
+typedef struct Frames {
+    uint16_t cnt;
+    uint16_t dyn_size;     /**< size in elements of `dframes` */
+    uint32_t progress_rel; /**< processing depth relative to STREAM_BASE_OFFSET */
+    uint64_t base_id;
+    uint32_t left_edge_rel;
+    Frame sframes[FRAMES_STATIC_CNT]; /**< static frames */
+    Frame *dframes;                   /**< dynamically allocated space for more frames */
+#ifdef DEBUG
+    uint8_t ipproto;
+    AppProto alproto;
+#endif
+} Frames;
+// size 136
+
+typedef struct FramesContainer {
+    Frames toserver;
+    Frames toclient;
+} FramesContainer;
+// size 272
+
+void FramesFree(Frames *frames);
+void FramesPrune(Flow *f, Packet *p);
+
+Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
+        const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type);
+Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice,
+        const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type);
+Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice,
+        const uint64_t frame_start, const int64_t len, int dir, uint8_t frame_type);
+void AppLayerFrameDump(Flow *f);
+
+Frame *FrameGetByIndex(Frames *frames, const uint32_t idx);
+Frame *FrameGetById(Frames *frames, const int64_t id);
+
+Frame *AppLayerFrameGetById(Flow *f, const int direction, const FrameId frame_id);
+FrameId AppLayerFrameGetId(Frame *r);
+void AppLayerFrameAddEvent(Frame *frame, uint8_t e);
+void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e);
+void AppLayerFrameSetLength(Frame *frame, int64_t len);
+void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_t len);
+void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id);
+void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id);
+
+void AppLayerFramesUpdateProgress(
+        Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction);
+void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction);
+
+FramesContainer *AppLayerFramesGetContainer(Flow *f);
+FramesContainer *AppLayerFramesSetupContainer(Flow *f);
+
+#endif
diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 806d5e249e..e4fb82d2e1 100644
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -165,8 +165,56 @@ struct AppLayerParserState_ {
 
     /* Used to store decoder events. */
     AppLayerDecoderEvents *decoder_events;
+
+    FramesContainer *frames;
 };
 
+static void AppLayerParserFramesFreeContainer(FramesContainer *frames)
+{
+    if (frames != NULL) {
+        FramesFree(&frames->toserver);
+        FramesFree(&frames->toclient);
+        SCFree(frames);
+    }
+}
+
+void AppLayerFramesFreeContainer(Flow *f)
+{
+    if (f == NULL || f->alparser == NULL || f->alparser->frames == NULL)
+        return;
+    AppLayerParserFramesFreeContainer(f->alparser->frames);
+    f->alparser->frames = NULL;
+}
+
+FramesContainer *AppLayerFramesGetContainer(Flow *f)
+{
+    if (f == NULL || f->alparser == NULL)
+        return NULL;
+    return f->alparser->frames;
+}
+
+FramesContainer *AppLayerFramesSetupContainer(Flow *f)
+{
+#ifdef UNITTESTS
+    if (f == NULL || f->alparser == NULL || f->protoctx == NULL)
+        return NULL;
+#endif
+    DEBUG_VALIDATE_BUG_ON(f == NULL || f->alparser == NULL);
+    if (f->alparser->frames == NULL) {
+        f->alparser->frames = SCCalloc(1, sizeof(FramesContainer));
+        if (f->alparser->frames == NULL) {
+            return NULL;
+        }
+#ifdef DEBUG
+        f->alparser->frames->toserver.ipproto = f->proto;
+        f->alparser->frames->toserver.alproto = f->alproto;
+        f->alparser->frames->toclient.ipproto = f->proto;
+        f->alparser->frames->toclient.alproto = f->alproto;
+#endif
+    }
+    return f->alparser->frames;
+}
+
 #ifdef UNITTESTS
 void UTHAppLayerParserStateGetIds(void *ptr, uint64_t *i1, uint64_t *i2, uint64_t *log, uint64_t *min)
 {
@@ -208,6 +256,7 @@ void AppLayerParserStateFree(AppLayerParserState *pstate)
 
     if (pstate->decoder_events != NULL)
         AppLayerDecoderEventsFreeEvents(&pstate->decoder_events);
+    AppLayerParserFramesFreeContainer(pstate->frames);
     SCFree(pstate);
 
     SCReturn;
diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index ec585ee394..daeaa31431 100644
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -26,6 +26,7 @@
 #define __APP_LAYER_PARSER_H__
 
 #include "app-layer-events.h"
+#include "app-layer-frames.h"
 #include "detect-engine-state.h"
 #include "util-file.h"
 #include "stream-tcp-private.h"
@@ -315,4 +316,6 @@ void AppLayerParserRestoreParserTable(void);
 void UTHAppLayerParserStateGetIds(void *ptr, uint64_t *i1, uint64_t *i2, uint64_t *log, uint64_t *min);
 #endif
 
+void AppLayerFramesFreeContainer(Flow *f);
+
 #endif /* __APP_LAYER_PARSER_H__ */
diff --git a/src/app-layer.c b/src/app-layer.c
index 6fc0339e1f..dbe390e2df 100644
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -32,6 +32,7 @@
 #include "app-layer-expectation.h"
 #include "app-layer-ftp.h"
 #include "app-layer-detect-proto.h"
+#include "app-layer-frames.h"
 #include "stream-tcp-reassemble.h"
 #include "stream-tcp-private.h"
 #include "stream-tcp-inline.h"
diff --git a/src/flow-worker.c b/src/flow-worker.c
index 984b81a875..81c1cdd98c 100644
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -428,6 +428,7 @@ static void FlowWorkerFlowTimeout(ThreadVars *tv, Packet *p, FlowWorkerThreadDat
     /* Prune any stored files. */
     FlowPruneFiles(p);
 
+    FramesPrune(p->flow, p);
     /*  Release tcp segments. Done here after alerting can use them. */
     FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
     StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
@@ -568,6 +569,7 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
                 StreamTcpSessionCleanup(p->flow->protoctx);
             }
         } else if (p->proto == IPPROTO_TCP && p->flow->protoctx) {
+            FramesPrune(p->flow, p);
             FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
             StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
                     STREAM_TOSERVER : STREAM_TOCLIENT);
diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c
index 5800251d17..cc9204abcf 100644
--- a/src/stream-tcp-list.c
+++ b/src/stream-tcp-list.c
@@ -31,6 +31,7 @@
 #include "util-streaming-buffer.h"
 #include "util-print.h"
 #include "util-validate.h"
+#include "app-layer-frames.h"
 
 static void StreamTcpRemoveSegmentFromStream(TcpStream *stream, TcpSegment *seg);
 
@@ -676,7 +677,24 @@ static inline bool StreamTcpReturnSegmentCheck(const TcpStream *stream, const Tc
     SCReturnInt(true);
 }
 
-static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
+static inline uint64_t GetLeftEdgeForApp(Flow *f, TcpSession *ssn, TcpStream *stream)
+{
+    const FramesContainer *frames_container = AppLayerFramesGetContainer(f);
+    if (frames_container == NULL)
+        return STREAM_APP_PROGRESS(stream);
+
+    const Frames *frames =
+            stream == &ssn->client ? &frames_container->toserver : &frames_container->toclient;
+    //    const uint64_t x = FramesLeftEdge(stream, frames);
+    //  BUG_ON(x != (frames->left_edge_rel + STREAM_BASE_OFFSET(stream)));
+    //    return x;
+    const uint64_t o = (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream);
+    SCLogDebug(
+            "%s: frames left edge: %" PRIu64, &ssn->client == stream ? "toserver" : "toclient", o);
+    return o;
+}
+
+static inline uint64_t GetLeftEdge(Flow *f, TcpSession *ssn, TcpStream *stream)
 {
     bool use_app = true;
     bool use_raw = true;
@@ -694,6 +712,8 @@ static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
         use_log = false;
     }
 
+    SCLogDebug("use_app %d use_raw %d use_log %d", use_app, use_raw, use_log);
+
     if (use_raw) {
         uint64_t raw_progress = STREAM_RAW_PROGRESS(stream);
 
@@ -721,18 +741,19 @@ static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
         }
 
         if (use_app) {
-            left_edge = MIN(STREAM_APP_PROGRESS(stream), raw_progress);
-            SCLogDebug("left_edge %"PRIu64", using both app:%"PRIu64", raw:%"PRIu64,
-                    left_edge, STREAM_APP_PROGRESS(stream), raw_progress);
+            const uint64_t app_le = GetLeftEdgeForApp(f, ssn, stream);
+            left_edge = MIN(app_le, raw_progress);
+            SCLogDebug("left_edge %" PRIu64 ", using both app:%" PRIu64 ", raw:%" PRIu64, left_edge,
+                    app_le, raw_progress);
         } else {
             left_edge = raw_progress;
             SCLogDebug("left_edge %"PRIu64", using only raw:%"PRIu64,
                     left_edge, raw_progress);
         }
     } else if (use_app) {
-        left_edge = STREAM_APP_PROGRESS(stream);
-        SCLogDebug("left_edge %"PRIu64", using only app:%"PRIu64,
-                left_edge, STREAM_APP_PROGRESS(stream));
+        const uint64_t app_le = GetLeftEdgeForApp(f, ssn, stream);
+        left_edge = app_le;
+        SCLogDebug("left_edge %" PRIu64 ", using only app:%" PRIu64, left_edge, app_le);
     } else {
         left_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
         SCLogDebug("no app & raw: left_edge %"PRIu64" (full stream)", left_edge);
@@ -847,10 +868,14 @@ void StreamTcpPruneSession(Flow *f, uint8_t flags)
         return;
     }
 
-    const uint64_t left_edge = GetLeftEdge(ssn, stream);
+    const uint64_t left_edge = GetLeftEdge(f, ssn, stream);
     if (left_edge && left_edge > STREAM_BASE_OFFSET(stream)) {
         uint32_t slide = left_edge - STREAM_BASE_OFFSET(stream);
         SCLogDebug("buffer sliding %u to offset %"PRIu64, slide, left_edge);
+
+        if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
+            AppLayerFramesSlide(f, slide, flags & (STREAM_TOSERVER | STREAM_TOCLIENT));
+        }
         StreamingBufferSlideToOffset(&stream->sb, left_edge);
         stream->base_seq += slide;
 
diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index ad2e30310d..99f0912d1d 100644
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -61,6 +61,7 @@
 #include "app-layer.h"
 #include "app-layer-events.h"
 #include "app-layer-parser.h"
+#include "app-layer-frames.h"
 
 #include "detect-engine-state.h"
 
@@ -1199,6 +1200,7 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv,
         (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
                 (uint8_t *)mydata, mydata_len, flags);
         AppLayerProfilingStore(ra_ctx->app_tctx, p);
+        AppLayerFrameDump(p->flow);
         uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
         if (new_app_progress == app_progress || FlowChangeProto(p->flow))
             break;
@@ -1388,6 +1390,14 @@ void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t prog
         stream = &ssn->server;
     }
 
+    /* Record updates */
+    if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
+        if (progress > STREAM_APP_PROGRESS(stream)) {
+            AppLayerFramesUpdateProgress(p->flow, stream, progress,
+                    PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
+        }
+    }
+
     if (progress > STREAM_RAW_PROGRESS(stream)) {
         uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
         stream->raw_progress_rel += slide;