files: tracking flag update

Improve flow file flags and file flags updates. Introduce a mask
that is set at start up to avoid lots of runtime checks.

Disable cocci flags check as it doesn't support the more dynamic
nature of the flag updates.

diff --git a/src/detect.c b/src/detect.c
index f5a4962b928e50b033e2e1126d76c6ec501e1bb2..2530dc87a6b4dcfca10f4b91d1170bce3d5aec84 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -386,54 +386,49 @@ DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_c
 
 /** \internal
  *  \brief update flow's file tracking flags based on the detection engine
+ *         A set of flags is prepared that is sent to the File API. The
+           File API may reject one or more based on the global force settings.
  */
 static inline void
-DetectPostInspectFileFlagsUpdate(Flow *pflow, const SigGroupHead *sgh, uint8_t direction)
+DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direction)
 {
-    /* see if this sgh requires us to consider file storing */
-    if (!FileForceFilestore() && (sgh == NULL ||
-                sgh->filestore_cnt == 0))
-    {
-        FileDisableStoring(pflow, direction);
-    }
+    uint16_t flow_file_flags = FLOWFILE_INIT;
+
+    if (sgh == NULL) {
+        SCLogDebug("requesting disabling all file features for flow");
+        flow_file_flags = FLOWFILE_NONE;
+    } else {
+        if (sgh->filestore_cnt == 0) {
+            SCLogDebug("requesting disabling filestore for flow");
+            flow_file_flags |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC);
+        }
 #ifdef HAVE_MAGIC
-    /* see if this sgh requires us to consider file magic */
-    if (!FileForceMagic() && (sgh == NULL ||
-                !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
-    {
-        SCLogDebug("disabling magic for flow");
-        FileDisableMagic(pflow, direction);
-    }
+        if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)) {
+            SCLogDebug("requesting disabling magic for flow");
+            flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC);
+        }
 #endif
-    /* see if this sgh requires us to consider file md5 */
-    if (!FileForceMd5() && (sgh == NULL ||
-                !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)))
-    {
-        SCLogDebug("disabling md5 for flow");
-        FileDisableMd5(pflow, direction);
-    }
-
-    /* see if this sgh requires us to consider file sha1 */
-    if (!FileForceSha1() && (sgh == NULL ||
-                !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)))
-    {
-        SCLogDebug("disabling sha1 for flow");
-        FileDisableSha1(pflow, direction);
-    }
-
-    /* see if this sgh requires us to consider file sha256 */
-    if (!FileForceSha256() && (sgh == NULL ||
-                !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)))
-    {
-        SCLogDebug("disabling sha256 for flow");
-        FileDisableSha256(pflow, direction);
+#ifdef HAVE_NSS
+        if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) {
+            SCLogDebug("requesting disabling md5 for flow");
+            flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC);
+        }
+        if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)) {
+            SCLogDebug("requesting disabling sha1 for flow");
+            flow_file_flags |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC);
+        }
+        if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)) {
+            SCLogDebug("requesting disabling sha256 for flow");
+            flow_file_flags |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC);
+        }
+#endif
+        if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) {
+            SCLogDebug("requesting disabling filesize for flow");
+            flow_file_flags |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC);
+        }
     }
-
-    /* see if this sgh requires us to consider filesize */
-    if (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE))
-    {
-        SCLogDebug("disabling filesize for flow");
-        FileDisableFilesize(pflow, direction);
+    if (flow_file_flags != 0) {
+        FileUpdateFlowFileFlags(f, flow_file_flags, direction);
     }
 }
 

diff --git a/src/flow.h b/src/flow.h
index b2eb217964834aa1a9265764a981e0fb5d00c04f..25365c69e654f3748e65fa07a9decc44de19a564 100644 (file)
--- a/src/flow.h
+++ b/src/flow.h
@@ -107,6 +107,8 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 
 /* File flags */
 
+#define FLOWFILE_INIT                   0
+
 /** no magic on files in this flow */
 #define FLOWFILE_NO_MAGIC_TS            BIT_U16(0)
 #define FLOWFILE_NO_MAGIC_TC            BIT_U16(1)
@@ -130,6 +132,19 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 #define FLOWFILE_NO_SIZE_TS             BIT_U16(10)
 #define FLOWFILE_NO_SIZE_TC             BIT_U16(11)
 
+#define FLOWFILE_NONE_TS (FLOWFILE_NO_MAGIC_TS | \
+                          FLOWFILE_NO_STORE_TS | \
+                          FLOWFILE_NO_MD5_TS   | \
+                          FLOWFILE_NO_SHA1_TS  | \
+                          FLOWFILE_NO_SHA256_TS| \
+                          FLOWFILE_NO_SIZE_TS)
+#define FLOWFILE_NONE_TC (FLOWFILE_NO_MAGIC_TC | \
+                          FLOWFILE_NO_STORE_TC | \
+                          FLOWFILE_NO_MD5_TC   | \
+                          FLOWFILE_NO_SHA1_TC  | \
+                          FLOWFILE_NO_SHA256_TC| \
+                          FLOWFILE_NO_SIZE_TC)
+#define FLOWFILE_NONE    (FLOWFILE_NONE_TS|FLOWFILE_NONE_TC)
 
 #define FLOW_IS_IPV4(f) \
     (((f)->flags & FLOW_IPV4) == FLOW_IPV4)
@@ -379,7 +394,6 @@ typedef struct Flow_
     uint32_t flags;         /**< generic flags */
 
     uint16_t file_flags;    /**< file tracking/extraction flags */
-    /* coccinelle: Flow:file_flags:FLOWFILE_ */
 
     /** destination port to be used in protocol detection. This is meant
      *  for use with STARTTLS and HTTP CONNECT detection */

diff --git a/src/util-file.c b/src/util-file.c
index fd11244c3007aa34a1721c4ce05de766841fb7da..509a53268923348bba886e61626866782102a0cf 100644 (file)
--- a/src/util-file.c
+++ b/src/util-file.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2012 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -39,6 +39,12 @@
 
 extern int g_detect_disabled;
 
+/** \brief mask of file flags we'll not set
+ *  This mask is set based on global file settings and
+ *  cannot be overriden by detection.
+ */
+static uint16_t g_file_flow_mask = 0;
+
 /** \brief switch to force filestore on all files
  *         regardless of the rules.
  */
@@ -88,26 +94,31 @@ static void FileEndSha256(File *ff);
 void FileForceFilestoreEnable(void)
 {
     g_file_force_filestore = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC);
 }
 
 void FileForceMagicEnable(void)
 {
     g_file_force_magic = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC);
 }
 
 void FileForceMd5Enable(void)
 {
     g_file_force_md5 = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC);
 }
 
 void FileForceSha1Enable(void)
 {
     g_file_force_sha1 = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC);
 }
 
 void FileForceSha256Enable(void)
 {
     g_file_force_sha256 = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC);
 }
 
 int FileForceFilestore(void)
@@ -152,6 +163,7 @@ int FileForceSha256(void)
 void FileForceTrackingEnable(void)
 {
     g_file_force_tracking = 1;
+    g_file_flow_mask |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC);
 }
 
 /**
@@ -1032,216 +1044,79 @@ int FileCloseFileById(FileContainer *ffc, uint32_t track_id,
     SCReturnInt(-1);
 }
 
-/**
- *  \brief disable file storage for a flow
+/** \brief set a flow's file flags
+ *  \param set_file_flags flags in both directions that are requested to set
  *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
+ *  This function will ignore the flags for the irrelevant direction and
+ *  also mask the flags with the global settings.
  */
-void FileDisableStoring(Flow *f, uint8_t direction)
+void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction)
 {
-    File *ptr = NULL;
-
     SCEnter();
-
     DEBUG_ASSERT_FLOW_LOCKED(f);
 
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_STORE_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_STORE_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            /* if we're already storing, we'll continue */
-            if (!(ptr->flags & FILE_STORE)) {
-                SCLogDebug("not storing this file");
-                ptr->flags |= FILE_NOSTORE;
-            }
-        }
-    }
-    SCReturn;
-}
-
-/**
- *  \brief disable file magic lookups for this flow
- *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
- */
-void FileDisableMagic(Flow *f, uint8_t direction)
-{
-    File *ptr = NULL;
-
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_MAGIC_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_MAGIC_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            SCLogDebug("disabling magic for file %p from direction %s",
-                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
-            ptr->flags |= FILE_NOMAGIC;
-        }
+    /* remove flags not in our direction and
+       don't disable what is globally enabled */
+    if (direction == STREAM_TOSERVER) {
+        set_file_flags &= ~(FLOWFILE_NONE_TC|g_file_flow_mask);
+    } else {
+        set_file_flags &= ~(FLOWFILE_NONE_TS|g_file_flow_mask);
     }
+    f->file_flags |= set_file_flags;
 
-    SCReturn;
-}
+    SCLogDebug("f->file_flags %04x set_file_flags %04x g_file_flow_mask %04x",
+            f->file_flags, set_file_flags, g_file_flow_mask);
 
-/**
- *  \brief disable file md5 calc for this flow
- *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
- */
-void FileDisableMd5(Flow *f, uint8_t direction)
-{
-    File *ptr = NULL;
-
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_MD5_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_MD5_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            SCLogDebug("disabling md5 for file %p from direction %s",
-                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
-            ptr->flags |= FILE_NOMD5;
-
-#ifdef HAVE_NSS
-            /* destroy any ctx we may have so far */
-            if (ptr->md5_ctx != NULL) {
-                HASH_Destroy(ptr->md5_ctx);
-                ptr->md5_ctx = NULL;
-            }
+    if (set_file_flags != 0 && f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) {
+        uint16_t per_file_flags = 0;
+#ifdef HAVE_MAGIC
+        if (set_file_flags & (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC))
+            per_file_flags |= FILE_NOMAGIC;
 #endif
-        }
-    }
-
-    SCReturn;
-}
-
-/**
- *  \brief disable file sha1 calc for this flow
- *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
-*/
-void FileDisableSha1(Flow *f, uint8_t direction)
-{
-    File *ptr = NULL;
-
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_SHA1_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_SHA1_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            SCLogDebug("disabling sha1 for file %p from direction %s",
-                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
-            ptr->flags |= FILE_NOSHA1;
-
 #ifdef HAVE_NSS
-            /* destroy any ctx we may have so far */
-            if (ptr->sha1_ctx != NULL) {
-                HASH_Destroy(ptr->sha1_ctx);
-                ptr->sha1_ctx = NULL;
-            }
+        if (set_file_flags & (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC))
+            per_file_flags |= FILE_NOMD5;
+        if (set_file_flags & (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC))
+            per_file_flags |= FILE_NOSHA1;
+        if (set_file_flags & (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC))
+            per_file_flags |= FILE_NOSHA256;
 #endif
-        }
-    }
-
-    SCReturn;
-}
-
-/**
- *  \brief disable file sha256 calc for this flow
- *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
- */
-void FileDisableSha256(Flow *f, uint8_t direction)
-{
-    File *ptr = NULL;
+        if (set_file_flags & (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC))
+            per_file_flags |= FILE_NOTRACK;
+        if (set_file_flags & (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC))
+            per_file_flags |= FILE_NOSTORE;
 
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_SHA256_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_SHA256_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            SCLogDebug("disabling sha256 for file %p from direction %s",
-                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
-            ptr->flags |= FILE_NOSHA256;
+        FileContainer *ffc = AppLayerParserGetFiles(f, direction);
+        if (ffc != NULL) {
+            for (File *ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
+                ptr->flags |= per_file_flags;
 
 #ifdef HAVE_NSS
-            /* destroy any ctx we may have so far */
-            if (ptr->sha256_ctx != NULL) {
-                HASH_Destroy(ptr->sha256_ctx);
-                ptr->sha256_ctx = NULL;
-            }
+                /* destroy any ctx we may have so far */
+                if ((per_file_flags & FILE_NOSHA256) &&
+                        ptr->sha256_ctx != NULL)
+                {
+                    HASH_Destroy(ptr->sha256_ctx);
+                    ptr->sha256_ctx = NULL;
+                }
+                if ((per_file_flags & FILE_NOSHA1) &&
+                    ptr->sha1_ctx != NULL)
+                {
+                    HASH_Destroy(ptr->sha1_ctx);
+                    ptr->sha1_ctx = NULL;
+                }
+                if ((per_file_flags & FILE_NOMD5) &&
+                        ptr->md5_ctx != NULL)
+                {
+                    HASH_Destroy(ptr->md5_ctx);
+                    ptr->md5_ctx = NULL;
+                }
 #endif
+            }
         }
     }
-
-    SCReturn;
 }
 
-/**
- *  \brief disable file size tracking for this flow
- *
- *  \param f *LOCKED* flow
- *  \param direction flow direction
- */
-void FileDisableFilesize(Flow *f, uint8_t direction)
-{
-    File *ptr = NULL;
-
-    SCEnter();
-
-    DEBUG_ASSERT_FLOW_LOCKED(f);
-
-    if (direction == STREAM_TOSERVER)
-        f->file_flags |= FLOWFILE_NO_SIZE_TS;
-    else
-        f->file_flags |= FLOWFILE_NO_SIZE_TC;
-
-    FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction);
-    if (ffc != NULL) {
-        for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
-            SCLogDebug("disabling size tracking for file %p from direction %s",
-                    ptr, direction == STREAM_TOSERVER ? "toserver":"toclient");
-            ptr->flags |= FILE_NOTRACK;
-        }
-    }
-
-    SCReturn;
-}
 
 
 /**

diff --git a/src/util-file.h b/src/util-file.h
index d907fc14f166c8b64c80d4e4aaa509415f22d1ff..a4e3c6a10ee9b4550c20aefe2ad082ad7bf991ac 100644 (file)
--- a/src/util-file.h
+++ b/src/util-file.h
@@ -198,15 +198,6 @@ int FileStore(File *);
 int FileSetTx(File *, uint64_t txid);
 void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id);
 
-/**
- *  \brief disable file storage for a flow
- *
- *  \param f *LOCKED* flow
- */
-void FileDisableStoring(struct Flow_ *, uint8_t);
-
-void FileDisableFilesize(Flow *f, uint8_t direction);
-
 /**
  *  \brief disable file storing for a transaction
  *
@@ -223,22 +214,20 @@ int FileForceFilestore(void);
 void FileReassemblyDepthEnable(uint32_t size);
 uint32_t FileReassemblyDepth(void);
 
-void FileDisableMagic(Flow *f, uint8_t);
 void FileForceMagicEnable(void);
 int FileForceMagic(void);
 
-void FileDisableMd5(Flow *f, uint8_t);
 void FileForceMd5Enable(void);
 int FileForceMd5(void);
 
-void FileDisableSha1(Flow *f, uint8_t);
 void FileForceSha1Enable(void);
 int FileForceSha1(void);
 
-void FileDisableSha256(Flow *f, uint8_t);
 void FileForceSha256Enable(void);
 int FileForceSha256(void);
 
+void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction);
+
 void FileForceHashParseCfg(ConfNode *);
 
 void FileForceTrackingEnable(void);