file: optionally use detect tracking in pruning

When the file API is used to do content inspection (currently only
smtp does this), the detection should be considered while pruning
the file chunks.

This patch introduces a new flag for the file API: FILE_USE_DETECT

When it is used, 'FilePrune' will not remove chunks that are (partly)
beyond the File::content_inspected tracker.

When using this flag, it's important to realize that when the detect
engine is disabled or rules are not matching, content_inspected
might not get updated.

diff --git a/src/util-file.c b/src/util-file.c
index 7e3ba16113c14fbbca870f7bc0d493c3c3a7dbea..2b45f3360769214f534c490f13dc089db1026b89 100644 (file)
--- a/src/util-file.c
+++ b/src/util-file.c
@@ -173,6 +173,14 @@ static int FilePruneFile(File *file)
         SCLogDebug("fd %p", fd);
 
         if (file->flags & FILE_NOSTORE || fd->stored == 1) {
+            /* keep chunks in memory as long as we still need to
+             * inspect them or parts of them */
+            if (file->flags & FILE_USE_DETECT) {
+                uint64_t right_edge = fd->stream_offset + fd->len;
+                if (file->content_inspected < right_edge)
+                    break;
+            }
+
             file->chunks_head = fd->next;
             if (file->chunks_tail == fd)
                 file->chunks_tail = fd->next;
@@ -563,6 +571,10 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
         SCLogDebug("not doing md5 for this file");
         ff->flags |= FILE_NOMD5;
     }
+    if (flags & FILE_USE_DETECT) {
+        SCLogDebug("considering content_inspect tracker when pruning");
+        ff->flags |= FILE_USE_DETECT;
+    }
 
 #ifdef HAVE_NSS
     if (!(ff->flags & FILE_NOMD5) || g_file_force_md5) {

diff --git a/src/util-file.h b/src/util-file.h
index 4b590c2c389f05b8b86b96afc3156f058e5762b6..9d87b593df0955ea0b3f7635e8beb0633aa80d9e 100644 (file)
--- a/src/util-file.h
+++ b/src/util-file.h
@@ -38,6 +38,7 @@
 #define FILE_STORE      0x0040
 #define FILE_STORED     0x0080
 #define FILE_NOTRACK    0x0100 /**< track size of file */
+#define FILE_USE_DETECT 0x0200 /**< use content_inspected tracker */
 
 typedef enum FileState_ {
     FILE_STATE_NONE = 0,    /**< no state */
@@ -80,7 +81,8 @@ typedef struct File_ {
     uint64_t chunks_cnt_max;
 #endif
     uint64_t content_len_so_far;
-    uint64_t content_inspected;
+    uint64_t content_inspected;     /**< used in pruning if FILE_USE_DETECT
+                                     *   flag is set */
 } File;
 
 typedef struct FileContainer_ {