smtp/frames: initial frame support

Adds the following frames:

  command_line
  data
  response_line

The *_line frames are per line, so in multi-line responses each line
will have it's own frame.

Ticket: #4905.

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index b5eb04ba9e215c6d77e2b25d796d5a2dc55b5d28..063b55d8780eac42234f92f4e20a8dbcfaef8ebc 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -35,6 +35,7 @@
 #include "app-layer-detect-proto.h"
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
+#include "app-layer-frames.h"
 #include "app-layer-smtp.h"
 
 #include "util-enum.h"
@@ -153,6 +154,43 @@ SCEnumCharMap smtp_decoder_event_table[] = {
     { NULL, -1 },
 };
 
+enum SMTPFrameTypes {
+    SMTP_FRAME_COMMAND_LINE,
+    SMTP_FRAME_DATA,
+    SMTP_FRAME_RESPONSE_LINE,
+};
+
+SCEnumCharMap smtp_frame_table[] = {
+    {
+            "command_line",
+            SMTP_FRAME_COMMAND_LINE,
+    },
+    {
+            "data",
+            SMTP_FRAME_DATA,
+    },
+    {
+            "response_line",
+            SMTP_FRAME_RESPONSE_LINE,
+    },
+    { NULL, -1 },
+};
+
+static int SMTPGetFrameIdByName(const char *frame_name)
+{
+    int id = SCMapEnumNameToValue(frame_name, smtp_frame_table);
+    if (id < 0) {
+        return -1;
+    }
+    return id;
+}
+
+static const char *SMTPGetFrameNameById(const uint8_t frame_id)
+{
+    const char *name = SCMapEnumValueToName(frame_id, smtp_frame_table);
+    return name;
+}
+
 typedef struct SMTPThreadCtx_ {
     MpmThreadCtx *smtp_mpm_thread_ctx;
     PrefilterRuleStore *pmq;
@@ -468,8 +506,8 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file)
  * \retval -1 Either when we don't have any new lines to supply anymore or
  *            on failure.
  */
-static AppLayerResult SMTPGetLine(
-        SMTPState *state, SMTPInput *input, SMTPLine *line, uint16_t direction)
+static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state, SMTPInput *input,
+        SMTPLine *line, uint16_t direction)
 {
     SCEnter();
 
@@ -477,6 +515,26 @@ static AppLayerResult SMTPGetLine(
     if (input->len <= 0)
         return APP_LAYER_ERROR;
 
+    const uint8_t type = direction == 0 ? SMTP_FRAME_COMMAND_LINE : SMTP_FRAME_RESPONSE_LINE;
+    Frame *frame = AppLayerFrameGetLastOpenByType(f, direction, type);
+    if (frame == NULL) {
+        if (direction == 0 &&
+                !(state->current_command == SMTP_COMMAND_DATA &&
+                        (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_COMMAND_LINE);
+            /* can't set tx id before (possibly) creating it */
+
+        } else if (direction == 1) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 1, SMTP_FRAME_RESPONSE_LINE);
+            if (frame != NULL && state->curr_tx) {
+                AppLayerFrameSetTxId(frame, state->curr_tx->tx_id);
+            }
+        }
+    }
+    SCLogDebug("frame %p", frame);
+
     uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
     bool discard_till_lf = (direction == 0) ? state->discard_till_lf_ts : state->discard_till_lf_tc;
 
@@ -503,6 +561,11 @@ static AppLayerResult SMTPGetLine(
         input->len -= line->len;
         DEBUG_VALIDATE_BUG_ON((input->consumed + input->len) != input->orig_len);
         line->buf = input->buf + o_consumed;
+
+        if (frame != NULL) {
+            frame->len = (int64_t)line->len;
+        }
+
         if (line->len >= SMTP_LINE_BUFFER_LIMIT) {
             line->len = SMTP_LINE_BUFFER_LIMIT;
             line->delim_len = 0;
@@ -1044,12 +1107,23 @@ static int NoNewTx(SMTPState *state, const SMTPLine *line)
  *         -1 for errors and inconsistent states
  *         -2 if MIME state could not be allocated
  * */
-static int SMTPProcessRequest(
-        SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line)
+static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *pstate,
+        SMTPInput *input, const SMTPLine *line, const StreamSlice *slice)
 {
     SCEnter();
     SMTPTransaction *tx = state->curr_tx;
 
+    Frame *frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_COMMAND_LINE);
+    if (frame) {
+        frame->len = (int64_t)line->len;
+    } else {
+        if (!(state->current_command == SMTP_COMMAND_DATA &&
+                    (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) {
+            frame = AppLayerFrameNewByPointer(
+                    f, slice, line->buf, line->len, 0, SMTP_FRAME_COMMAND_LINE);
+        }
+    }
+
     /* If current input is to be discarded because it completes a long line,
      * line's length and delimiter len are reset to 0. Skip processing this line.
      * This line is only to get us out of the state where we should discard any
@@ -1070,6 +1144,9 @@ static int SMTPProcessRequest(
         StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER,
                 smtp_config.content_inspect_min_size);
     }
+    if (frame != NULL && state->curr_tx) {
+        AppLayerFrameSetTxId(frame, state->curr_tx->tx_id);
+    }
 
     state->toserver_data_count += (line->len + line->delim_len);
 
@@ -1107,6 +1184,15 @@ static int SMTPProcessRequest(
                 }
             }
             state->curr_tx->is_data = true;
+
+            Frame *data_frame = AppLayerFrameNewByPointer(
+                    f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_DATA);
+            if (data_frame == NULL) {
+                SCLogDebug("data_frame %p - no data frame set up", data_frame);
+            } else {
+                AppLayerFrameSetTxId(data_frame, state->curr_tx->tx_id);
+            }
+
             /* Enter immediately data mode without waiting for server reply */
             if (state->parser_state & SMTP_PARSER_STATE_PIPELINING_SERVER) {
                 state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
@@ -1201,8 +1287,8 @@ static inline void ResetLine(SMTPLine *line)
  *          1 for handing control over to GetLine
  *         -1 for errors and inconsistent states
  * */
-static int SMTPPreProcessCommands(
-        SMTPState *state, Flow *f, AppLayerParserState *pstate, SMTPInput *input, SMTPLine *line)
+static int SMTPPreProcessCommands(SMTPState *state, Flow *f, AppLayerParserState *pstate,
+        StreamSlice *slice, SMTPInput *input, SMTPLine *line)
 {
     DEBUG_VALIDATE_BUG_ON((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0);
     DEBUG_VALIDATE_BUG_ON(line->len != 0);
@@ -1251,10 +1337,11 @@ static int SMTPPreProcessCommands(
             if (line->len < 0) {
                 return -1;
             }
+
             input->consumed = total_consumed;
             input->len -= current_line_consumed;
             DEBUG_VALIDATE_BUG_ON(input->consumed + input->len != input->orig_len);
-            if (SMTPProcessRequest(state, f, pstate, line) == -1) {
+            if (SMTPProcessRequest(state, f, pstate, input, line, slice) == -1) {
                 return -1;
             }
             line_complete = false;
@@ -1263,8 +1350,13 @@ static int SMTPPreProcessCommands(
             line->delim_len = 0;
 
             /* bail if `SMTPProcessRequest` ended the data mode */
-            if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0)
+            if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0) {
+                Frame *data_frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_DATA);
+                if (data_frame) {
+                    data_frame->len = (slice->offset + input->consumed) - data_frame->offset;
+                }
                 break;
+            }
         }
     }
     return 0;
@@ -1295,7 +1387,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
         if (((state->current_command == SMTP_COMMAND_DATA) ||
                     (state->current_command == SMTP_COMMAND_BDAT)) &&
                 (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
-            int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line);
+            int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line);
             DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1);
             if (ret == 0 && input.consumed == input.orig_len) {
                 SCReturnStruct(APP_LAYER_OK);
@@ -1303,9 +1395,9 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                 SCReturnStruct(APP_LAYER_ERROR);
             }
         }
-        AppLayerResult res = SMTPGetLine(state, &input, &line, direction);
+        AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         while (res.status == 0) {
-            int retval = SMTPProcessRequest(state, f, pstate, &line);
+            int retval = SMTPProcessRequest(state, f, pstate, &input, &line, &stream_slice);
             if (retval != 0)
                 SCReturnStruct(APP_LAYER_ERROR);
             if (line.delim_len == 0 && line.len == SMTP_LINE_BUFFER_LIMIT) {
@@ -1326,7 +1418,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
              * In case of another boundary, the control should be passed to SMTPGetLine */
             if ((input.len > 0) && (state->current_command == SMTP_COMMAND_DATA) &&
                     (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
-                int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line);
+                int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line);
                 DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1);
                 if (ret == 0 && input.consumed == input.orig_len) {
                     SCReturnStruct(APP_LAYER_OK);
@@ -1334,13 +1426,13 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                     SCReturnStruct(APP_LAYER_ERROR);
                 }
             }
-            res = SMTPGetLine(state, &input, &line, direction);
+            res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         }
         if (res.status == 1)
             return res;
         /* toclient */
     } else {
-        AppLayerResult res = SMTPGetLine(state, &input, &line, direction);
+        AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         while (res.status == 0) {
             if (SMTPProcessReply(state, f, pstate, thread_data, &input, &line) != 0)
                 SCReturnStruct(APP_LAYER_ERROR);
@@ -1352,7 +1444,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
                 SMTPSetEvent(state, SMTP_DECODER_EVENT_TRUNCATED_LINE);
                 break;
             }
-            res = SMTPGetLine(state, &input, &line, direction);
+            res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction);
         }
         if (res.status == 1)
             return res;
@@ -1752,6 +1844,8 @@ void RegisterSMTPParsers(void)
         AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData);
         AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData);
         AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1);
+        AppLayerParserRegisterGetFrameFuncs(
+                IPPROTO_TCP, ALPROTO_SMTP, SMTPGetFrameIdByName, SMTPGetFrameNameById);
     } else {
         SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name);
     }

diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h
index 93c3bd812c939f5495a4e67e0f74335bdd8488e9..cd9c614b966a0fc4bcac53105584ce55923af608 100644 (file)
--- a/src/app-layer-smtp.h
+++ b/src/app-layer-smtp.h
@@ -24,6 +24,7 @@
 #ifndef SURICATA_APP_LAYER_SMTP_H
 #define SURICATA_APP_LAYER_SMTP_H
 
+#include "app-layer-frames.h"
 #include "util-streaming-buffer.h"
 #include "rust.h"