From: Shivani Bhardwaj <shivanib134@gmail.com>
Date: Mon, 3 Jan 2022 14:14:53 +0000 (+0530)
Subject: smtp: use AppLayerResult instead of buffering
X-Git-Tag: suricata-7.0.0-beta1~999
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F6819%2Fhead;p=thirdparty%2Fsuricata.git

smtp: use AppLayerResult instead of buffering

Also, remove tests that check for the removed buffers and any middle
states while parsing and buffering.

Ticket 4907
---

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 175ad6ac6b..54eed744b2 100644
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -570,220 +570,33 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
  * \retval -1 Either when we don't have any new lines to supply anymore or
  *            on failure.
  */
-static int SMTPGetLine(SMTPState *state)
+static AppLayerResult SMTPGetLine(SMTPState *state)
 {
     SCEnter();
-    void *ptmp;
 
     /* we have run out of input */
     if (state->input_len <= 0)
-        return -1;
-
-    /* toserver */
-    if (state->direction == 0) {
-        if (state->ts_current_line_lf_seen == 1) {
-            /* we have seen the lf for the previous line.  Clear the parser
-             * details to parse new line */
-            state->ts_current_line_lf_seen = 0;
-            if (state->ts_current_line_db == 1) {
-                state->ts_current_line_db = 0;
-                SCFree(state->ts_db);
-                state->ts_db = NULL;
-                state->ts_db_len = 0;
-                state->current_line = NULL;
-                state->current_line_len = 0;
-            }
-        }
-
-        uint8_t *lf_idx = memchr(state->input, 0x0a, state->input_len);
-
-        if (lf_idx == NULL) {
-            /* fragmented lines.  Decoder event for special cases.  Not all
-             * fragmented lines should be treated as a possible evasion
-             * attempt.  With multi payload smtp chunks we can have valid
-             * cases of fragmentation.  But within the same segment chunk
-             * if we see fragmentation then it's definitely something you
-             * should alert about */
-            if (state->ts_current_line_db == 0) {
-                state->ts_db = SCMalloc(state->input_len);
-                if (state->ts_db == NULL) {
-                    return -1;
-                }
-                state->ts_current_line_db = 1;
-                memcpy(state->ts_db, state->input, state->input_len);
-                state->ts_db_len = state->input_len;
-            } else {
-                ptmp = SCRealloc(state->ts_db,
-                                 (state->ts_db_len + state->input_len));
-                if (ptmp == NULL) {
-                    SCFree(state->ts_db);
-                    state->ts_db = NULL;
-                    state->ts_db_len = 0;
-                    return -1;
-                }
-                state->ts_db = ptmp;
-
-                memcpy(state->ts_db + state->ts_db_len,
-                       state->input, state->input_len);
-                state->ts_db_len += state->input_len;
-            } /* else */
-            state->input += state->input_len;
-            state->input_len = 0;
-
-            return -1;
-
-        } else {
-            state->ts_current_line_lf_seen = 1;
-
-            if (state->ts_current_line_db == 1) {
-                ptmp = SCRealloc(state->ts_db,
-                                 (state->ts_db_len + (lf_idx + 1 - state->input)));
-                if (ptmp == NULL) {
-                    SCFree(state->ts_db);
-                    state->ts_db = NULL;
-                    state->ts_db_len = 0;
-                    return -1;
-                }
-                state->ts_db = ptmp;
-
-                memcpy(state->ts_db + state->ts_db_len,
-                       state->input, (lf_idx + 1 - state->input));
-                state->ts_db_len += (lf_idx + 1 - state->input);
-
-                if (state->ts_db_len > 1 &&
-                    state->ts_db[state->ts_db_len - 2] == 0x0D) {
-                    state->ts_db_len -= 2;
-                    state->current_line_delimiter_len = 2;
-                } else {
-                    state->ts_db_len -= 1;
-                    state->current_line_delimiter_len = 1;
-                }
-
-                state->current_line = state->ts_db;
-                state->current_line_len = state->ts_db_len;
+        return APP_LAYER_ERROR;
 
-            } else {
-                state->current_line = state->input;
-                state->current_line_len = lf_idx - state->input;
-
-                if (state->input != lf_idx &&
-                    *(lf_idx - 1) == 0x0D) {
-                    state->current_line_len--;
-                    state->current_line_delimiter_len = 2;
-                } else {
-                    state->current_line_delimiter_len = 1;
-                }
-            }
-
-            state->input_len -= (lf_idx - state->input) + 1;
-            state->input = (lf_idx + 1);
-
-            return 0;
-        }
+    uint8_t *lf_idx = memchr(state->input + state->consumed, 0x0a, state->input_len);
 
-        /* toclient */
+    if (lf_idx == NULL) {
+        SCReturnStruct(APP_LAYER_INCOMPLETE(state->consumed, state->input_len + 1));
     } else {
-        if (state->tc_current_line_lf_seen == 1) {
-            /* we have seen the lf for the previous line.  Clear the parser
-             * details to parse new line */
-            state->tc_current_line_lf_seen = 0;
-            if (state->tc_current_line_db == 1) {
-                state->tc_current_line_db = 0;
-                SCFree(state->tc_db);
-                state->tc_db = NULL;
-                state->tc_db_len = 0;
-                state->current_line = NULL;
-                state->current_line_len = 0;
-            }
-        }
-
-        uint8_t *lf_idx = memchr(state->input, 0x0a, state->input_len);
-
-        if (lf_idx == NULL) {
-            /* fragmented lines.  Decoder event for special cases.  Not all
-             * fragmented lines should be treated as a possible evasion
-             * attempt.  With multi payload smtp chunks we can have valid
-             * cases of fragmentation.  But within the same segment chunk
-             * if we see fragmentation then it's definitely something you
-             * should alert about */
-            if (state->tc_current_line_db == 0) {
-                state->tc_db = SCMalloc(state->input_len);
-                if (state->tc_db == NULL) {
-                    return -1;
-                }
-                state->tc_current_line_db = 1;
-                memcpy(state->tc_db, state->input, state->input_len);
-                state->tc_db_len = state->input_len;
-            } else {
-                ptmp = SCRealloc(state->tc_db,
-                                 (state->tc_db_len + state->input_len));
-                if (ptmp == NULL) {
-                    SCFree(state->tc_db);
-                    state->tc_db = NULL;
-                    state->tc_db_len = 0;
-                    return -1;
-                }
-                state->tc_db = ptmp;
-
-                memcpy(state->tc_db + state->tc_db_len,
-                       state->input, state->input_len);
-                state->tc_db_len += state->input_len;
-            } /* else */
-            state->input += state->input_len;
-            state->input_len = 0;
-
-            return -1;
-
+        uint32_t o_consumed = state->consumed;
+        state->consumed = lf_idx - state->input + 1;
+        state->current_line_len = state->consumed - o_consumed;
+        state->current_line = state->input + o_consumed;
+        state->input_len -= state->current_line_len;
+        if (state->consumed >= 2 && state->input[state->consumed - 2] == 0x0D) {
+            state->current_line_delimiter_len = 2;
+            state->current_line_len -= 2;
         } else {
-            state->tc_current_line_lf_seen = 1;
-
-            if (state->tc_current_line_db == 1) {
-                ptmp = SCRealloc(state->tc_db,
-                                 (state->tc_db_len + (lf_idx + 1 - state->input)));
-                if (ptmp == NULL) {
-                    SCFree(state->tc_db);
-                    state->tc_db = NULL;
-                    state->tc_db_len = 0;
-                    return -1;
-                }
-                state->tc_db = ptmp;
-
-                memcpy(state->tc_db + state->tc_db_len,
-                       state->input, (lf_idx + 1 - state->input));
-                state->tc_db_len += (lf_idx + 1 - state->input);
-
-                if (state->tc_db_len > 1 &&
-                    state->tc_db[state->tc_db_len - 2] == 0x0D) {
-                    state->tc_db_len -= 2;
-                    state->current_line_delimiter_len = 2;
-                } else {
-                    state->tc_db_len -= 1;
-                    state->current_line_delimiter_len = 1;
-                }
-
-                state->current_line = state->tc_db;
-                state->current_line_len = state->tc_db_len;
-
-            } else {
-                state->current_line = state->input;
-                state->current_line_len = lf_idx - state->input;
-
-                if (state->input != lf_idx &&
-                    *(lf_idx - 1) == 0x0D) {
-                    state->current_line_len--;
-                    state->current_line_delimiter_len = 2;
-                } else {
-                    state->current_line_delimiter_len = 1;
-                }
-            }
-
-            state->input_len -= (lf_idx - state->input) + 1;
-            state->input = (lf_idx + 1);
-
-            return 0;
-        } /* else - if (lf_idx == NULL) */
+            state->current_line_delimiter_len = 1;
+            state->current_line_len -= 1;
+        }
+        SCReturnStruct(APP_LAYER_OK);
     }
-
 }
 
 static int SMTPInsertCommandIntoCommandBuffer(uint8_t command, SMTPState *state, Flow *f)
@@ -1406,21 +1219,28 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
 
     state->input = input;
     state->input_len = input_len;
+    state->consumed = 0;
     state->direction = direction;
+    AppLayerResult res = SMTPGetLine(state);
 
     /* toserver */
     if (direction == 0) {
-        while (SMTPGetLine(state) >= 0) {
+        while (res.status == 0) {
             if (SMTPProcessRequest(state, f, pstate) == -1)
                 SCReturnStruct(APP_LAYER_ERROR);
+            res = SMTPGetLine(state);
         }
-
+        if (res.status == 1)
+            return res;
         /* toclient */
     } else {
-        while (SMTPGetLine(state) >= 0) {
+        while (res.status == 0) {
             if (SMTPProcessReply(state, f, pstate, thread_data) == -1)
                 SCReturnStruct(APP_LAYER_ERROR);
+            res = SMTPGetLine(state);
         }
+        if (res.status == 1)
+            return res;
     }
 
     SCReturnStruct(APP_LAYER_OK);
@@ -1565,12 +1385,6 @@ static void SMTPStateFree(void *p)
     if (smtp_state->cmds != NULL) {
         SCFree(smtp_state->cmds);
     }
-    if (smtp_state->ts_current_line_db) {
-        SCFree(smtp_state->ts_db);
-    }
-    if (smtp_state->tc_current_line_db) {
-        SCFree(smtp_state->tc_db);
-    }
 
     if (smtp_state->helo) {
         SCFree(smtp_state->helo);
@@ -1935,10 +1749,8 @@ static int SMTPParserTest01(void)
         printf("no smtp state: ");
         goto end;
     }
-    if (smtp_state->input_len != 0 ||
-        smtp_state->cmds_cnt != 0 ||
-        smtp_state->cmds_idx != 0 ||
-        smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
+    if (smtp_state->cmds_cnt != 0 || smtp_state->cmds_idx != 0 ||
+            smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
         printf("smtp parser in inconsistent state\n");
         goto end;
     }
@@ -1952,11 +1764,9 @@ static int SMTPParserTest01(void)
         goto end;
     }
     FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->input_len != 0 ||
-        smtp_state->cmds_cnt != 1 ||
-        smtp_state->cmds_idx != 0 ||
-        smtp_state->cmds[0] != SMTP_COMMAND_OTHER_CMD ||
-        smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
+    if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 ||
+            smtp_state->cmds[0] != SMTP_COMMAND_OTHER_CMD ||
+            smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
         printf("smtp parser in inconsistent state\n");
         goto end;
     }
@@ -1970,10 +1780,8 @@ static int SMTPParserTest01(void)
         goto end;
     }
     FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->input_len != 0 ||
-        smtp_state->cmds_cnt != 0 ||
-        smtp_state->cmds_idx != 0 ||
-        smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
+    if (smtp_state->cmds_cnt != 0 || smtp_state->cmds_idx != 0 ||
+            smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
         printf("smtp parser in inconsistent state\n");
         goto end;
     }
@@ -1987,11 +1795,9 @@ static int SMTPParserTest01(void)
         goto end;
     }
     FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->input_len != 0 ||
-        smtp_state->cmds_cnt != 1 ||
-        smtp_state->cmds_idx != 0 ||
-        smtp_state->cmds[0] != SMTP_COMMAND_STARTTLS ||
-        smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
+    if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 ||
+            smtp_state->cmds[0] != SMTP_COMMAND_STARTTLS ||
+            smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) {
         printf("smtp parser in inconsistent state\n");
         goto end;
     }
@@ -2005,11 +1811,9 @@ static int SMTPParserTest01(void)
         goto end;
     }
     FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->input_len != 0 ||
-        smtp_state->cmds_cnt != 0 ||
-        smtp_state->cmds_idx != 0 ||
-        smtp_state->parser_state != (SMTP_PARSER_STATE_FIRST_REPLY_SEEN |
-                                     SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
+    if (smtp_state->cmds_cnt != 0 || smtp_state->cmds_idx != 0 ||
+            smtp_state->parser_state !=
+                    (SMTP_PARSER_STATE_FIRST_REPLY_SEEN | SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
         printf("smtp parser in inconsistent state\n");
         goto end;
     }
@@ -3714,556 +3518,6 @@ end:
     return result;
 }
 
-/*
- * \test Test retrieving lines when frag'ed.
- */
-static int SMTPParserTest07(void)
-{
-    int result = 0;
-    Flow f;
-    int r = 0;
-
-    const char *request1_str = "EHLO boo.com";
-    /* EHLO boo.com<CR> */
-    uint8_t request1_1[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d,
-    };
-    int32_t request1_1_len = sizeof(request1_1);
-
-    /* <LF> */
-    uint8_t request1_2[] = {
-        0x0a
-    };
-    int32_t request1_2_len = sizeof(request1_2);
-
-    /* EHLO boo.com<CR><LF> */
-    uint8_t request2[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d, 0x0a,
-    };
-    int32_t request2_len = sizeof(request2);
-
-    TcpSession ssn;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    f.proto = IPPROTO_TCP;
-    f.alproto = ALPROTO_SMTP;
-
-    StreamTcpInitConfig(true);
-    SMTPTestInitConfig();
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_1, request1_1_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    SMTPState *smtp_state = f.alstate;
-    if (smtp_state == NULL) {
-        printf("no smtp state: ");
-        goto end;
-    }
-    if (smtp_state->current_line != NULL ||
-        smtp_state->current_line_len != 0 ||
-        smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != request1_1_len ||
-        memcmp(smtp_state->ts_db, request1_1, request1_1_len) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_2, request1_2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->ts_db, request1_str, strlen(request1_str)) != 0 ||
-        smtp_state->current_line != smtp_state->ts_db ||
-        smtp_state->current_line_len != smtp_state->ts_db_len) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request2, request2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 0 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->current_line, request1_str, strlen(request1_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    result = 1;
-end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    StreamTcpFreeConfig(true);
-    FLOW_DESTROY(&f);
-    return result;
-}
-
-/*
- * \test Test retrieving lines when frag'ed.
- */
-static int SMTPParserTest08(void)
-{
-    int result = 0;
-    Flow f;
-    int r = 0;
-
-    const char *request1_str = "EHLO boo.com";
-    /* EHLO boo.com */
-    uint8_t request1_1[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d,
-    };
-    int32_t request1_1_len = sizeof(request1_1);
-
-    /* <CR><LF> */
-    uint8_t request1_2[] = {
-        0x0d, 0x0a
-    };
-    int32_t request1_2_len = sizeof(request1_2);
-
-    /* EHLO boo.com<CR><LF> */
-    uint8_t request2[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d, 0x0a,
-    };
-    int32_t request2_len = sizeof(request2);
-
-    TcpSession ssn;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    f.proto = IPPROTO_TCP;
-    f.alproto = ALPROTO_SMTP;
-
-    StreamTcpInitConfig(true);
-    SMTPTestInitConfig();
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_1, request1_1_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    SMTPState *smtp_state = f.alstate;
-    if (smtp_state == NULL) {
-        printf("no smtp state: ");
-        goto end;
-    }
-    if (smtp_state->current_line != NULL ||
-        smtp_state->current_line_len != 0 ||
-        smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != request1_1_len ||
-        memcmp(smtp_state->ts_db, request1_1, request1_1_len) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_2, request1_2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->ts_db, request1_str, strlen(request1_str)) != 0 ||
-        smtp_state->current_line != smtp_state->ts_db ||
-        smtp_state->current_line_len != smtp_state->ts_db_len) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request2, request2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 0 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->current_line, request1_str, strlen(request1_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    result = 1;
-end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    StreamTcpFreeConfig(true);
-    FLOW_DESTROY(&f);
-    return result;
-}
-
-/*
- * \test Test retrieving lines when frag'ed.
- */
-static int SMTPParserTest09(void)
-{
-    int result = 0;
-    Flow f;
-    int r = 0;
-
-    const char *request1_str = "EHLO boo.com";
-    /* EHLO boo. */
-    uint8_t request1_1[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e,
-    };
-    int32_t request1_1_len = sizeof(request1_1);
-
-    /* com<CR><LF> */
-    uint8_t request1_2[] = {
-        0x63, 0x6f, 0x6d, 0x0d, 0x0a
-    };
-    int32_t request1_2_len = sizeof(request1_2);
-
-    /* EHLO boo.com<CR><LF> */
-    uint8_t request2[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d, 0x0a,
-    };
-    int32_t request2_len = sizeof(request2);
-
-    TcpSession ssn;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    f.proto = IPPROTO_TCP;
-    f.alproto = ALPROTO_SMTP;
-
-    StreamTcpInitConfig(true);
-    SMTPTestInitConfig();
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_1, request1_1_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    SMTPState *smtp_state = f.alstate;
-    if (smtp_state == NULL) {
-        printf("no smtp state: ");
-        goto end;
-    }
-    if (smtp_state->current_line != NULL ||
-        smtp_state->current_line_len != 0 ||
-        smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != request1_1_len ||
-        memcmp(smtp_state->ts_db, request1_1, request1_1_len) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_2, request1_2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->ts_db, request1_str, strlen(request1_str)) != 0 ||
-        smtp_state->current_line != smtp_state->ts_db ||
-        smtp_state->current_line_len != smtp_state->ts_db_len) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request2, request2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 0 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->current_line, request1_str, strlen(request1_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    result = 1;
-end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    StreamTcpFreeConfig(true);
-    FLOW_DESTROY(&f);
-    return result;
-}
-
-/*
- * \test Test retrieving lines when frag'ed.
- */
-static int SMTPParserTest10(void)
-{
-    int result = 0;
-    Flow f;
-    int r = 0;
-
-    const char *request1_str = "";
-    /* EHLO boo. */
-    uint8_t request1_1[] = {
-        0x0d,
-    };
-    int32_t request1_1_len = sizeof(request1_1);
-
-    /* com<CR><LF> */
-    uint8_t request1_2[] = {
-        0x0a,
-    };
-    int32_t request1_2_len = sizeof(request1_2);
-
-    const char *request2_str = "EHLO boo.com";
-    /* EHLO boo.com<CR><LF> */
-    uint8_t request2[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d, 0x0a,
-    };
-    int32_t request2_len = sizeof(request2);
-
-    TcpSession ssn;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    f.proto = IPPROTO_TCP;
-    f.alproto = ALPROTO_SMTP;
-
-    StreamTcpInitConfig(true);
-    SMTPTestInitConfig();
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_1, request1_1_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    SMTPState *smtp_state = f.alstate;
-    if (smtp_state == NULL) {
-        printf("no smtp state: ");
-        goto end;
-    }
-    if (smtp_state->current_line != NULL ||
-        smtp_state->current_line_len != 0 ||
-        smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != request1_1_len ||
-        memcmp(smtp_state->ts_db, request1_1, request1_1_len) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1_2, request1_2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 1 ||
-        smtp_state->ts_db == NULL ||
-        smtp_state->ts_db_len != (int32_t)strlen(request1_str) ||
-        memcmp(smtp_state->ts_db, request1_str, strlen(request1_str)) != 0 ||
-        smtp_state->current_line != smtp_state->ts_db ||
-        smtp_state->current_line_len != smtp_state->ts_db_len) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request2, request2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 0 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != (int32_t)strlen(request2_str) ||
-        memcmp(smtp_state->current_line, request2_str, strlen(request2_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    result = 1;
-end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    StreamTcpFreeConfig(true);
-    FLOW_DESTROY(&f);
-    return result;
-}
-
-/*
- * \test Test retrieving lines when frag'ed.
- */
-static int SMTPParserTest11(void)
-{
-    int result = 0;
-    Flow f;
-    int r = 0;
-
-    const char *request1_str = "";
-    /* EHLO boo. */
-    uint8_t request1[] = {
-        0x0a,
-    };
-    int32_t request1_len = sizeof(request1);
-
-    const char *request2_str = "EHLO boo.com";
-    /* EHLO boo.com<CR><LF> */
-    uint8_t request2[] = {
-        0x45, 0x48, 0x4c, 0x4f, 0x20, 0x62, 0x6f, 0x6f,
-        0x2e, 0x63, 0x6f, 0x6d, 0x0d, 0x0a,
-    };
-    int32_t request2_len = sizeof(request2);
-
-    TcpSession ssn;
-    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
-
-    memset(&f, 0, sizeof(f));
-    memset(&ssn, 0, sizeof(ssn));
-
-    FLOW_INITIALIZE(&f);
-    f.protoctx = (void *)&ssn;
-    f.proto = IPPROTO_TCP;
-    f.alproto = ALPROTO_SMTP;
-
-    StreamTcpInitConfig(true);
-    SMTPTestInitConfig();
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request1, request1_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    SMTPState *smtp_state = f.alstate;
-    if (smtp_state == NULL) {
-        printf("no smtp state: ");
-        goto end;
-    }
-    if (smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != 0 ||
-        smtp_state->ts_current_line_db == 1 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        memcmp(smtp_state->current_line, request1_str, strlen(request1_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    FLOWLOCK_WRLOCK(&f);
-    r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
-                            STREAM_TOSERVER, request2, request2_len);
-    if (r != 0) {
-        printf("smtp check returned %" PRId32 ", expected 0: ", r);
-        FLOWLOCK_UNLOCK(&f);
-        goto end;
-    }
-    FLOWLOCK_UNLOCK(&f);
-    if (smtp_state->ts_current_line_db != 0 ||
-        smtp_state->ts_db != NULL ||
-        smtp_state->ts_db_len != 0 ||
-        smtp_state->current_line == NULL ||
-        smtp_state->current_line_len != (int32_t)strlen(request2_str) ||
-        memcmp(smtp_state->current_line, request2_str, strlen(request2_str)) != 0) {
-        printf("smtp parser in inconsistent state\n");
-        goto end;
-    }
-
-    result = 1;
-end:
-    if (alp_tctx != NULL)
-        AppLayerParserThreadCtxFree(alp_tctx);
-    StreamTcpFreeConfig(true);
-    FLOW_DESTROY(&f);
-    return result;
-}
-
 static int SMTPParserTest12(void)
 {
     int result = 0;
@@ -5296,11 +4550,6 @@ void SMTPParserRegisterTests(void)
     UtRegisterTest("SMTPParserTest04", SMTPParserTest04);
     UtRegisterTest("SMTPParserTest05", SMTPParserTest05);
     UtRegisterTest("SMTPParserTest06", SMTPParserTest06);
-    UtRegisterTest("SMTPParserTest07", SMTPParserTest07);
-    UtRegisterTest("SMTPParserTest08", SMTPParserTest08);
-    UtRegisterTest("SMTPParserTest09", SMTPParserTest09);
-    UtRegisterTest("SMTPParserTest10", SMTPParserTest10);
-    UtRegisterTest("SMTPParserTest11", SMTPParserTest11);
     UtRegisterTest("SMTPParserTest12", SMTPParserTest12);
     UtRegisterTest("SMTPParserTest13", SMTPParserTest13);
     UtRegisterTest("SMTPParserTest14", SMTPParserTest14);
diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h
index 649cf4852c..d86e4da62e 100644
--- a/src/app-layer-smtp.h
+++ b/src/app-layer-smtp.h
@@ -119,22 +119,8 @@ typedef struct SMTPState_ {
     /** length of the line in current_line.  Doesn't include the delimiter */
     int32_t current_line_len;
     uint8_t current_line_delimiter_len;
-
-    /** used to indicate if the current_line buffer is a malloced buffer.  We
-     * use a malloced buffer, if a line is fragmented */
-    uint8_t *tc_db;
-    int32_t tc_db_len;
-    uint8_t tc_current_line_db;
-    /** we have see LF for the currently parsed line */
-    uint8_t tc_current_line_lf_seen;
-
-    /** used to indicate if the current_line buffer is a malloced buffer.  We
-     * use a malloced buffer, if a line is fragmented */
-    uint8_t *ts_db;
-    int32_t ts_db_len;
-    uint8_t ts_current_line_db;
-    /** we have see LF for the currently parsed line */
-    uint8_t ts_current_line_lf_seen;
+    /* Consumed bytes till current line */
+    int32_t consumed;
 
     /** var to indicate parser state */
     uint8_t parser_state;