From: Victor Julien <victor@inliniac.net>
Date: Wed, 9 Mar 2016 19:43:54 +0000 (+0100)
Subject: file: optionally use detect tracking in pruning
X-Git-Tag: suricata-3.0.1RC1~15
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=77358a4113e4fab82464a257e95484f8dcb25d21;p=thirdparty%2Fsuricata.git

file: optionally use detect tracking in pruning

When the file API is used to do content inspection (currently only
smtp does this), the detection should be considered while pruning
the file chunks.

This patch introduces a new flag for the file API: FILE_USE_DETECT

When it is used, 'FilePrune' will not remove chunks that are (partly)
beyond the File::content_inspected tracker.

When using this flag, it's important to realize that when the detect
engine is disabled or rules are not matching, content_inspected
might not get updated.
---

diff --git a/src/util-file.c b/src/util-file.c
index 7e3ba16113..2b45f33607 100644
--- a/src/util-file.c
+++ b/src/util-file.c
@@ -173,6 +173,14 @@ static int FilePruneFile(File *file)
         SCLogDebug("fd %p", fd);
 
         if (file->flags & FILE_NOSTORE || fd->stored == 1) {
+            /* keep chunks in memory as long as we still need to
+             * inspect them or parts of them */
+            if (file->flags & FILE_USE_DETECT) {
+                uint64_t right_edge = fd->stream_offset + fd->len;
+                if (file->content_inspected < right_edge)
+                    break;
+            }
+
             file->chunks_head = fd->next;
             if (file->chunks_tail == fd)
                 file->chunks_tail = fd->next;
@@ -563,6 +571,10 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
         SCLogDebug("not doing md5 for this file");
         ff->flags |= FILE_NOMD5;
     }
+    if (flags & FILE_USE_DETECT) {
+        SCLogDebug("considering content_inspect tracker when pruning");
+        ff->flags |= FILE_USE_DETECT;
+    }
 
 #ifdef HAVE_NSS
     if (!(ff->flags & FILE_NOMD5) || g_file_force_md5) {
diff --git a/src/util-file.h b/src/util-file.h
index 4b590c2c38..9d87b593df 100644
--- a/src/util-file.h
+++ b/src/util-file.h
@@ -38,6 +38,7 @@
 #define FILE_STORE      0x0040
 #define FILE_STORED     0x0080
 #define FILE_NOTRACK    0x0100 /**< track size of file */
+#define FILE_USE_DETECT 0x0200 /**< use content_inspected tracker */
 
 typedef enum FileState_ {
     FILE_STATE_NONE = 0,    /**< no state */
@@ -80,7 +81,8 @@ typedef struct File_ {
     uint64_t chunks_cnt_max;
 #endif
     uint64_t content_len_so_far;
-    uint64_t content_inspected;
+    uint64_t content_inspected;     /**< used in pruning if FILE_USE_DETECT
+                                     *   flag is set */
 } File;
 
 typedef struct FileContainer_ {