smtp: exit data mode if data command was rejected

And the server was advertising pipelining.

Ticket: 6906
(cherry picked from commit ed895c04ff1239bd75f97a1fd7a4b53706f7f25c)

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 3465e63256dfd98cd3c71e8ae8187dd62fc4884b..05843a9bb20a908bc0e39e059acbf096f68b67db 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -993,6 +993,10 @@ static int SMTPProcessReply(SMTPState *state, Flow *f, AppLayerParserState *psta
             state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
         } else {
             /* decoder event */
+            if (state->parser_state & SMTP_PARSER_STATE_PIPELINING_SERVER) {
+                // reset data mode if we had entered it prematurely
+                state->parser_state &= ~SMTP_PARSER_STATE_COMMAND_DATA_MODE;
+            }
             SMTPSetEvent(state, SMTP_DECODER_EVENT_DATA_COMMAND_REJECTED);
         }
     } else if (IsReplyToCommand(state, SMTP_COMMAND_RSET)) {