From: Victor Julien Date: Tue, 26 Feb 2019 11:38:19 +0000 (+0100) Subject: files: tracking flag update X-Git-Tag: suricata-6.0.0-beta1~773 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=500e8da63aba55e8a892d081eaf5de8d81b23dbf;p=thirdparty%2Fsuricata.git files: tracking flag update Improve flow file flags and file flags updates. Introduce a mask that is set at start up to avoid lots of runtime checks. Disable cocci flags check as it doesn't support the more dynamic nature of the flag updates. --- diff --git a/src/detect.c b/src/detect.c index f5a4962b92..2530dc87a6 100644 --- a/src/detect.c +++ b/src/detect.c @@ -386,54 +386,49 @@ DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_c /** \internal * \brief update flow's file tracking flags based on the detection engine + * A set of flags is prepared that is sent to the File API. The + File API may reject one or more based on the global force settings. */ static inline void -DetectPostInspectFileFlagsUpdate(Flow *pflow, const SigGroupHead *sgh, uint8_t direction) +DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direction) { - /* see if this sgh requires us to consider file storing */ - if (!FileForceFilestore() && (sgh == NULL || - sgh->filestore_cnt == 0)) - { - FileDisableStoring(pflow, direction); - } + uint16_t flow_file_flags = FLOWFILE_INIT; + + if (sgh == NULL) { + SCLogDebug("requesting disabling all file features for flow"); + flow_file_flags = FLOWFILE_NONE; + } else { + if (sgh->filestore_cnt == 0) { + SCLogDebug("requesting disabling filestore for flow"); + flow_file_flags |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC); + } #ifdef HAVE_MAGIC - /* see if this sgh requires us to consider file magic */ - if (!FileForceMagic() && (sgh == NULL || - !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC))) - { - SCLogDebug("disabling magic for flow"); - FileDisableMagic(pflow, direction); - } + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)) { + SCLogDebug("requesting disabling magic for flow"); + flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC); + } #endif - /* see if this sgh requires us to consider file md5 */ - if (!FileForceMd5() && (sgh == NULL || - !(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5))) - { - SCLogDebug("disabling md5 for flow"); - FileDisableMd5(pflow, direction); - } - - /* see if this sgh requires us to consider file sha1 */ - if (!FileForceSha1() && (sgh == NULL || - !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1))) - { - SCLogDebug("disabling sha1 for flow"); - FileDisableSha1(pflow, direction); - } - - /* see if this sgh requires us to consider file sha256 */ - if (!FileForceSha256() && (sgh == NULL || - !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256))) - { - SCLogDebug("disabling sha256 for flow"); - FileDisableSha256(pflow, direction); +#ifdef HAVE_NSS + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) { + SCLogDebug("requesting disabling md5 for flow"); + flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC); + } + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)) { + SCLogDebug("requesting disabling sha1 for flow"); + flow_file_flags |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC); + } + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)) { + SCLogDebug("requesting disabling sha256 for flow"); + flow_file_flags |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC); + } +#endif + if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) { + SCLogDebug("requesting disabling filesize for flow"); + flow_file_flags |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC); + } } - - /* see if this sgh requires us to consider filesize */ - if (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) - { - SCLogDebug("disabling filesize for flow"); - FileDisableFilesize(pflow, direction); + if (flow_file_flags != 0) { + FileUpdateFlowFileFlags(f, flow_file_flags, direction); } } diff --git a/src/flow.h b/src/flow.h index b2eb217964..25365c69e6 100644 --- a/src/flow.h +++ b/src/flow.h @@ -107,6 +107,8 @@ typedef struct AppLayerParserState_ AppLayerParserState; /* File flags */ +#define FLOWFILE_INIT 0 + /** no magic on files in this flow */ #define FLOWFILE_NO_MAGIC_TS BIT_U16(0) #define FLOWFILE_NO_MAGIC_TC BIT_U16(1) @@ -130,6 +132,19 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOWFILE_NO_SIZE_TS BIT_U16(10) #define FLOWFILE_NO_SIZE_TC BIT_U16(11) +#define FLOWFILE_NONE_TS (FLOWFILE_NO_MAGIC_TS | \ + FLOWFILE_NO_STORE_TS | \ + FLOWFILE_NO_MD5_TS | \ + FLOWFILE_NO_SHA1_TS | \ + FLOWFILE_NO_SHA256_TS| \ + FLOWFILE_NO_SIZE_TS) +#define FLOWFILE_NONE_TC (FLOWFILE_NO_MAGIC_TC | \ + FLOWFILE_NO_STORE_TC | \ + FLOWFILE_NO_MD5_TC | \ + FLOWFILE_NO_SHA1_TC | \ + FLOWFILE_NO_SHA256_TC| \ + FLOWFILE_NO_SIZE_TC) +#define FLOWFILE_NONE (FLOWFILE_NONE_TS|FLOWFILE_NONE_TC) #define FLOW_IS_IPV4(f) \ (((f)->flags & FLOW_IPV4) == FLOW_IPV4) @@ -379,7 +394,6 @@ typedef struct Flow_ uint32_t flags; /**< generic flags */ uint16_t file_flags; /**< file tracking/extraction flags */ - /* coccinelle: Flow:file_flags:FLOWFILE_ */ /** destination port to be used in protocol detection. This is meant * for use with STARTTLS and HTTP CONNECT detection */ diff --git a/src/util-file.c b/src/util-file.c index fd11244c30..509a532689 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2012 Open Information Security Foundation +/* Copyright (C) 2007-2020 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -39,6 +39,12 @@ extern int g_detect_disabled; +/** \brief mask of file flags we'll not set + * This mask is set based on global file settings and + * cannot be overriden by detection. + */ +static uint16_t g_file_flow_mask = 0; + /** \brief switch to force filestore on all files * regardless of the rules. */ @@ -88,26 +94,31 @@ static void FileEndSha256(File *ff); void FileForceFilestoreEnable(void) { g_file_force_filestore = 1; + g_file_flow_mask |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC); } void FileForceMagicEnable(void) { g_file_force_magic = 1; + g_file_flow_mask |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC); } void FileForceMd5Enable(void) { g_file_force_md5 = 1; + g_file_flow_mask |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC); } void FileForceSha1Enable(void) { g_file_force_sha1 = 1; + g_file_flow_mask |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC); } void FileForceSha256Enable(void) { g_file_force_sha256 = 1; + g_file_flow_mask |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC); } int FileForceFilestore(void) @@ -152,6 +163,7 @@ int FileForceSha256(void) void FileForceTrackingEnable(void) { g_file_force_tracking = 1; + g_file_flow_mask |= (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC); } /** @@ -1032,216 +1044,79 @@ int FileCloseFileById(FileContainer *ffc, uint32_t track_id, SCReturnInt(-1); } -/** - * \brief disable file storage for a flow +/** \brief set a flow's file flags + * \param set_file_flags flags in both directions that are requested to set * - * \param f *LOCKED* flow - * \param direction flow direction + * This function will ignore the flags for the irrelevant direction and + * also mask the flags with the global settings. */ -void FileDisableStoring(Flow *f, uint8_t direction) +void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction) { - File *ptr = NULL; - SCEnter(); - DEBUG_ASSERT_FLOW_LOCKED(f); - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_STORE_TS; - else - f->file_flags |= FLOWFILE_NO_STORE_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - /* if we're already storing, we'll continue */ - if (!(ptr->flags & FILE_STORE)) { - SCLogDebug("not storing this file"); - ptr->flags |= FILE_NOSTORE; - } - } - } - SCReturn; -} - -/** - * \brief disable file magic lookups for this flow - * - * \param f *LOCKED* flow - * \param direction flow direction - */ -void FileDisableMagic(Flow *f, uint8_t direction) -{ - File *ptr = NULL; - - SCEnter(); - - DEBUG_ASSERT_FLOW_LOCKED(f); - - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_MAGIC_TS; - else - f->file_flags |= FLOWFILE_NO_MAGIC_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - SCLogDebug("disabling magic for file %p from direction %s", - ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); - ptr->flags |= FILE_NOMAGIC; - } + /* remove flags not in our direction and + don't disable what is globally enabled */ + if (direction == STREAM_TOSERVER) { + set_file_flags &= ~(FLOWFILE_NONE_TC|g_file_flow_mask); + } else { + set_file_flags &= ~(FLOWFILE_NONE_TS|g_file_flow_mask); } + f->file_flags |= set_file_flags; - SCReturn; -} + SCLogDebug("f->file_flags %04x set_file_flags %04x g_file_flow_mask %04x", + f->file_flags, set_file_flags, g_file_flow_mask); -/** - * \brief disable file md5 calc for this flow - * - * \param f *LOCKED* flow - * \param direction flow direction - */ -void FileDisableMd5(Flow *f, uint8_t direction) -{ - File *ptr = NULL; - - SCEnter(); - - DEBUG_ASSERT_FLOW_LOCKED(f); - - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_MD5_TS; - else - f->file_flags |= FLOWFILE_NO_MD5_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - SCLogDebug("disabling md5 for file %p from direction %s", - ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); - ptr->flags |= FILE_NOMD5; - -#ifdef HAVE_NSS - /* destroy any ctx we may have so far */ - if (ptr->md5_ctx != NULL) { - HASH_Destroy(ptr->md5_ctx); - ptr->md5_ctx = NULL; - } + if (set_file_flags != 0 && f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) { + uint16_t per_file_flags = 0; +#ifdef HAVE_MAGIC + if (set_file_flags & (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC)) + per_file_flags |= FILE_NOMAGIC; #endif - } - } - - SCReturn; -} - -/** - * \brief disable file sha1 calc for this flow - * - * \param f *LOCKED* flow - * \param direction flow direction -*/ -void FileDisableSha1(Flow *f, uint8_t direction) -{ - File *ptr = NULL; - - SCEnter(); - - DEBUG_ASSERT_FLOW_LOCKED(f); - - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_SHA1_TS; - else - f->file_flags |= FLOWFILE_NO_SHA1_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - SCLogDebug("disabling sha1 for file %p from direction %s", - ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); - ptr->flags |= FILE_NOSHA1; - #ifdef HAVE_NSS - /* destroy any ctx we may have so far */ - if (ptr->sha1_ctx != NULL) { - HASH_Destroy(ptr->sha1_ctx); - ptr->sha1_ctx = NULL; - } + if (set_file_flags & (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC)) + per_file_flags |= FILE_NOMD5; + if (set_file_flags & (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC)) + per_file_flags |= FILE_NOSHA1; + if (set_file_flags & (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC)) + per_file_flags |= FILE_NOSHA256; #endif - } - } - - SCReturn; -} - -/** - * \brief disable file sha256 calc for this flow - * - * \param f *LOCKED* flow - * \param direction flow direction - */ -void FileDisableSha256(Flow *f, uint8_t direction) -{ - File *ptr = NULL; + if (set_file_flags & (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC)) + per_file_flags |= FILE_NOTRACK; + if (set_file_flags & (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC)) + per_file_flags |= FILE_NOSTORE; - SCEnter(); - - DEBUG_ASSERT_FLOW_LOCKED(f); - - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_SHA256_TS; - else - f->file_flags |= FLOWFILE_NO_SHA256_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - SCLogDebug("disabling sha256 for file %p from direction %s", - ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); - ptr->flags |= FILE_NOSHA256; + FileContainer *ffc = AppLayerParserGetFiles(f, direction); + if (ffc != NULL) { + for (File *ptr = ffc->head; ptr != NULL; ptr = ptr->next) { + ptr->flags |= per_file_flags; #ifdef HAVE_NSS - /* destroy any ctx we may have so far */ - if (ptr->sha256_ctx != NULL) { - HASH_Destroy(ptr->sha256_ctx); - ptr->sha256_ctx = NULL; - } + /* destroy any ctx we may have so far */ + if ((per_file_flags & FILE_NOSHA256) && + ptr->sha256_ctx != NULL) + { + HASH_Destroy(ptr->sha256_ctx); + ptr->sha256_ctx = NULL; + } + if ((per_file_flags & FILE_NOSHA1) && + ptr->sha1_ctx != NULL) + { + HASH_Destroy(ptr->sha1_ctx); + ptr->sha1_ctx = NULL; + } + if ((per_file_flags & FILE_NOMD5) && + ptr->md5_ctx != NULL) + { + HASH_Destroy(ptr->md5_ctx); + ptr->md5_ctx = NULL; + } #endif + } } } - - SCReturn; } -/** - * \brief disable file size tracking for this flow - * - * \param f *LOCKED* flow - * \param direction flow direction - */ -void FileDisableFilesize(Flow *f, uint8_t direction) -{ - File *ptr = NULL; - - SCEnter(); - - DEBUG_ASSERT_FLOW_LOCKED(f); - - if (direction == STREAM_TOSERVER) - f->file_flags |= FLOWFILE_NO_SIZE_TS; - else - f->file_flags |= FLOWFILE_NO_SIZE_TC; - - FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - SCLogDebug("disabling size tracking for file %p from direction %s", - ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); - ptr->flags |= FILE_NOTRACK; - } - } - - SCReturn; -} /** diff --git a/src/util-file.h b/src/util-file.h index d907fc14f1..a4e3c6a10e 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -198,15 +198,6 @@ int FileStore(File *); int FileSetTx(File *, uint64_t txid); void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id); -/** - * \brief disable file storage for a flow - * - * \param f *LOCKED* flow - */ -void FileDisableStoring(struct Flow_ *, uint8_t); - -void FileDisableFilesize(Flow *f, uint8_t direction); - /** * \brief disable file storing for a transaction * @@ -223,22 +214,20 @@ int FileForceFilestore(void); void FileReassemblyDepthEnable(uint32_t size); uint32_t FileReassemblyDepth(void); -void FileDisableMagic(Flow *f, uint8_t); void FileForceMagicEnable(void); int FileForceMagic(void); -void FileDisableMd5(Flow *f, uint8_t); void FileForceMd5Enable(void); int FileForceMd5(void); -void FileDisableSha1(Flow *f, uint8_t); void FileForceSha1Enable(void); int FileForceSha1(void); -void FileDisableSha256(Flow *f, uint8_t); void FileForceSha256Enable(void); int FileForceSha256(void); +void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction); + void FileForceHashParseCfg(ConfNode *); void FileForceTrackingEnable(void);