smtp: use detection file tracking

As SMTP file_data detection uses the file API, the file's inspect
tracker should be considered when pruning files.

This patch sets the FILE_USE_DETECT flag on files tracked by smtp.

It also adds logic to move inspected tracker ahead if detection
doesn't do it, like when no rules are matching or detection engine
is disabled.

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index fd317c7e78dfd7f61e50fe4651026fec13965f5a..1dd6cab1c2caebcea44be1354db2deef9bfb4fa8 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -347,9 +347,46 @@ static SMTPTransaction *SMTPTransactionCreate(void)
     return tx;
 }
 
-int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
-        MimeDecParseState *state) {
+/** \internal
+ *  \brief update inspected tracker if it gets to far behind
+ *
+ *  As smtp uses the FILE_USE_DETECT flag in the file API, we are responsible
+ *  for making sure that File::content_inspected is not getting too far
+ *  behind.
+ */
+static void SMTPPruneFiles(FileContainer *files)
+{
+    SCLogDebug("cfg: win %"PRIu32" min_size %"PRIu32,
+            smtp_config.content_inspect_window, smtp_config.content_inspect_min_size);
+
+    File *file = files->head;
+    while (file) {
+        if (file->chunks_head) {
+            uint32_t window = smtp_config.content_inspect_window;
+            if (file->chunks_head->stream_offset == 0)
+                window = MAX(window, smtp_config.content_inspect_min_size);
+
+            uint64_t file_size = file->content_len_so_far;
+            uint64_t data_size = file_size - file->chunks_head->stream_offset;
+
+            SCLogDebug("window %"PRIu32", file_size %"PRIu64", data_size %"PRIu64,
+                    window, file_size, data_size);
+
+            if (data_size > (window * 3)) {
+                uint64_t left_edge = file_size - window;
+                SCLogDebug("file->content_inspected now %"PRIu64, left_edge);
+                file->content_inspected = left_edge;
+            }
+        }
+
+        file = file->next;
+    }
+}
 
+int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
+        MimeDecParseState *state)
+{
+    SCEnter();
     int ret = MIME_DEC_OK;
     Flow *flow = (Flow *) state->data;
     SMTPState *smtp_state = (SMTPState *) flow->alstate;
@@ -410,7 +447,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
             }
 
             if (FileOpenFile(files, (uint8_t *) entity->filename, entity->filename_len,
-                    (uint8_t *) chunk, len, flags) == NULL) {
+                    (uint8_t *) chunk, len, flags|FILE_USE_DETECT) == NULL) {
                 ret = MIME_DEC_ERR_DATA;
                 SCLogDebug("FileOpenFile() failed");
             }
@@ -466,6 +503,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
     }
 
     if (files != NULL) {
+        SMTPPruneFiles(files);
         FilePrune(files);
     }
 

diff --git a/src/detect-engine-filedata-smtp.c b/src/detect-engine-filedata-smtp.c
index 5b8f8dca4c8f8b22667a4669e21de60eb0f8dc45..f037dfec7a1afdddee44a9a083104daf3f6217c6 100644 (file)
--- a/src/detect-engine-filedata-smtp.c
+++ b/src/detect-engine-filedata-smtp.c
@@ -357,7 +357,7 @@ static int DetectEngineSMTPFiledataTest01(void)
     p->flow = &f;
     p->flowflags |= FLOW_PKT_TOSERVER;
     p->flowflags |= FLOW_PKT_ESTABLISHED;
-    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST|PKT_STREAM_EOF;
     f.alproto = ALPROTO_SMTP;
 
     StreamTcpInitConfig(TRUE);