smtp: trigger raw stream inspection

Internals
---------
Suricata's stream engine returns data for inspection to the detection
engine from the stream when the chunk size is reached.

Bug
---
Inspection triggered only in the specified chunk sizes may be too late
when it comes to inspection of smaller protocol specific data which
could result in delayed inspection, incorrect data logged with a transaction
and logs misindicating the pkt that triggered an alert.

Fix
---
Fix this by making an explicit call from all respective applayer parsers to
trigger raw stream inspection which shall make the data available for inspection
in the following call of the stream engine. This needs to happen per direction
on the completion of an entity like a request or a response.

Important notes
---------------
1. The above mentioned behavior with and without this patch is
affected internally by the following conditions.
- inspection depth
- stream depth
In these special cases, the inspection window will be affected and
Suricata may not consider all the data that could be expected to be
inspected.
2. This only applies to applayer protocols running over TCP.
3. The inspection window is only considered up to the ACK'd data.
4. This entire issue is about IDS mode only.

SMTP parser can handle multiple command lines per direction, however an
SMTP transaction comprises of the full communication starting from HELO
till there's a RST or QUIT request. Appropriate calls to trigger raw stream
inspection have been added on succesful parsing of each full request and response.

Task 7026
Bug 7004

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 0cb009ca8ef8a39c4286ef0a480aa87f0dd29a34..5dfa7737b730118cfe8fbd5d0199e64b7ed507f7 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -705,11 +705,13 @@ static void SetMimeEvents(SMTPState *state, uint32_t events)
     }
 }
 
-static inline void SMTPTransactionComplete(SMTPState *state)
+static inline void SMTPTransactionComplete(SMTPState *state, Flow *f, uint16_t dir)
 {
     DEBUG_VALIDATE_BUG_ON(state->curr_tx == NULL);
-    if (state->curr_tx)
+    if (state->curr_tx) {
         state->curr_tx->done = true;
+        AppLayerParserTriggerRawStreamInspection(f, dir);
+    }
 }
 
 /**
@@ -746,7 +748,7 @@ static int SMTPProcessCommandDATA(
                         FileFlowToFlags(f, STREAM_TOSERVER));
             }
         }
-        SMTPTransactionComplete(state);
+        SMTPTransactionComplete(state, f, STREAM_TOSERVER);
         SCLogDebug("marked tx as done");
     } else if (smtp_config.raw_extraction) {
         // message not over, store the line. This is a substitution of
@@ -931,7 +933,7 @@ static int SMTPProcessReply(
                 SMTPSetEvent(state, SMTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE);
             }
             if (state->curr_tx) {
-                SMTPTransactionComplete(state);
+                SMTPTransactionComplete(state, f, STREAM_TOCLIENT);
             }
         } else {
             /* decoder event */
@@ -952,7 +954,7 @@ static int SMTPProcessReply(
     } else if (IsReplyToCommand(state, SMTP_COMMAND_RSET)) {
         if (reply_code == SMTP_REPLY_250 && state->curr_tx &&
                 !(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) {
-            SMTPTransactionComplete(state);
+            SMTPTransactionComplete(state, f, STREAM_TOCLIENT);
         }
     } else {
         /* we don't care for any other command for now */
@@ -1093,12 +1095,14 @@ static int SMTPParseCommandRCPTTO(SMTPState *state, const SMTPLine *line)
 }
 
 /* consider 'rset' and 'quit' to be part of the existing state */
-static int NoNewTx(SMTPState *state, const SMTPLine *line)
+static int NoNewTx(SMTPState *state, Flow *f, const SMTPLine *line)
 {
     if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
         if (line->len >= 4 && SCMemcmpLowercase("rset", line->buf, 4) == 0) {
+            AppLayerParserTriggerRawStreamInspection(f, STREAM_TOSERVER);
             return 1;
         } else if (line->len >= 4 && SCMemcmpLowercase("quit", line->buf, 4) == 0) {
+            AppLayerParserTriggerRawStreamInspection(f, STREAM_TOSERVER);
             return 1;
         }
     }
@@ -1149,7 +1153,7 @@ static int SMTPProcessRequest(
     if (line->len == 0 && line->delim_len == 0) {
         return 0;
     }
-    if (state->curr_tx == NULL || (state->curr_tx->done && !NoNewTx(state, line))) {
+    if (state->curr_tx == NULL || (state->curr_tx->done && !NoNewTx(state, f, line))) {
         tx = SMTPTransactionCreate(state);
         if (tx == NULL)
             return -1;