From a6d928e269db561c223a31e5677f298708541009 Mon Sep 17 00:00:00 2001 From: Duarte Silva Date: Fri, 29 Apr 2016 21:51:12 +0200 Subject: [PATCH] file-hashing: added support for SHA-1 file hashing --- src/Makefile.am | 1 + src/app-layer-htp-file.c | 10 +++ src/app-layer-smtp.c | 4 + src/detect-engine-file.c | 6 ++ src/detect-engine-siggroup.c | 12 ++- src/detect-engine-siggroup.h | 2 +- src/detect-filesha1.c | 162 +++++++++++++++++++++++++++++++++++ src/detect-filesha1.h | 31 +++++++ src/detect.c | 39 ++++++++- src/detect.h | 13 +-- src/flow.h | 36 ++++---- src/log-file.c | 9 +- src/log-filestore.c | 8 ++ src/output-json-file.c | 9 ++ src/suricata.c | 2 +- src/util-error.c | 1 + src/util-error.h | 1 + src/util-file.c | 90 +++++++++++++++++-- src/util-file.h | 4 + src/util-lua-common.c | 12 +++ 20 files changed, 415 insertions(+), 37 deletions(-) create mode 100644 src/detect-filesha1.c create mode 100644 src/detect-filesha1.h diff --git a/src/Makefile.am b/src/Makefile.am index 6c2744829d..edc6d65d2e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -138,6 +138,7 @@ detect-file-data.c detect-file-data.h \ detect-fileext.c detect-fileext.h \ detect-filemagic.c detect-filemagic.h \ detect-filemd5.c detect-filemd5.h \ +detect-filesha1.c detect-filesha1.h \ detect-filename.c detect-filename.h \ detect-filesize.c detect-filesize.h \ detect-filestore.c detect-filestore.h \ diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index 4dcddf6865..18b193612b 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -118,6 +118,11 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len, flags |= FILE_NOMD5; } + if (s->f->flags & FLOW_FILE_NO_SHA1_TC) { + SCLogDebug("no sha1 for this flow in toclient direction, so none for this file"); + flags |= FILE_NOSHA1; + } + if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) { flags |= FILE_NOSTORE; } @@ -150,6 +155,11 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len, flags |= FILE_NOMD5; } + if (s->f->flags & FLOW_FILE_NO_SHA1_TS) { + SCLogDebug("no sha1 for this flow in toserver direction, so none for this file"); + flags |= FILE_NOSHA1; + } + if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) { flags |= FILE_NOSTORE; } diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 6c2aede9b3..f8b25c9470 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -423,6 +423,10 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, flags |= FILE_NOMD5; } + if (flow->flags & FLOW_FILE_NO_SHA1_TS) { + flags |= FILE_NOSHA1; + } + /* Find file */ if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index 1afe967d6a..5eb36e853d 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -129,6 +129,12 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, break; } + if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) { + SCLogDebug("sig needs file sha1, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + break; + } + if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) { SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED"); r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index a55b530468..58a3058ad2 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -516,7 +516,7 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, } /** - * \brief Set the need md5 flag in the sgh. + * \brief Set the need magic flag in the sgh. * * \param de_ctx detection engine ctx for the signatures * \param sgh sig group head to set the flag in @@ -608,12 +608,12 @@ void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } /** - * \brief Set the need magic flag in the sgh. + * \brief Set the need hash flag in the sgh. * * \param de_ctx detection engine ctx for the signatures * \param sgh sig group head to set the flag in */ -void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +void SigGroupHeadSetFileHashFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { Signature *s = NULL; uint32_t sig = 0; @@ -631,6 +631,12 @@ void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *de_ctx, SigGroupHead *sgh) SCLogDebug("sgh %p has filemd5", sgh); break; } + + if (SignatureIsFileSha1Inspecting(s)) { + sgh->flags |= SIG_GROUP_HEAD_HAVEFILESHA1; + SCLogDebug("sgh %p has filesha1", sgh); + break; + } } return; diff --git a/src/detect-engine-siggroup.h b/src/detect-engine-siggroup.h index 7b88bbc7c7..b8dd6b1c9f 100644 --- a/src/detect-engine-siggroup.h +++ b/src/detect-engine-siggroup.h @@ -67,7 +67,7 @@ void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *); -void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *); +void SigGroupHeadSetFileHashFlag(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilesizeFlag(DetectEngineCtx *, SigGroupHead *); uint16_t SigGroupHeadGetMinMpmSize(DetectEngineCtx *de_ctx, SigGroupHead *sgh, int list); diff --git a/src/detect-filesha1.c b/src/detect-filesha1.c new file mode 100644 index 0000000000..ea4e469bfb --- /dev/null +++ b/src/detect-filesha1.c @@ -0,0 +1,162 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * \author Duarte Silva + * + */ + +#include "suricata-common.h" + +#include "util-detect-file-hash.h" + +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#include "detect-filesha1.h" + +#ifndef HAVE_NSS + +static int DetectFileSha1SetupNoSupport (DetectEngineCtx *a, Signature *b, char *c) +{ + SCLogError(SC_ERR_NO_SHA1_SUPPORT, "no SHA-1 calculation support built in, needed for filesha1 keyword"); + return -1; +} + +/** + * \brief Registration function for keyword: filesha1 + */ +void DetectFileSha1Register(void) +{ + sigmatch_table[DETECT_FILESHA1].name = "filesha1"; + sigmatch_table[DETECT_FILESHA1].FileMatch = NULL; + sigmatch_table[DETECT_FILESHA1].Setup = DetectFileSha1SetupNoSupport; + sigmatch_table[DETECT_FILESHA1].Free = NULL; + sigmatch_table[DETECT_FILESHA1].RegisterTests = NULL; + sigmatch_table[DETECT_FILESHA1].flags = SIGMATCH_NOT_BUILT; + + SCLogDebug("registering filesha1 rule option"); + return; +} + +#else /* HAVE_NSS */ + +static int DetectFileSha1Setup (DetectEngineCtx *, Signature *, char *); +static void DetectFileSha1RegisterTests(void); + +/** + * \brief Registration function for keyword: filesha1 + */ +void DetectFileSha1Register(void) +{ + sigmatch_table[DETECT_FILESHA1].name = "filesha1"; + sigmatch_table[DETECT_FILESHA1].desc = "match file SHA-1 against list of SHA-1 checksums"; + sigmatch_table[DETECT_FILESHA1].url = "https://redmine.openinfosecfoundation.org/projects/suricata/wiki/File-keywords#filesha1"; + sigmatch_table[DETECT_FILESHA1].FileMatch = DetectFileHashMatch; + sigmatch_table[DETECT_FILESHA1].Setup = DetectFileSha1Setup; + sigmatch_table[DETECT_FILESHA1].Free = DetectFileHashFree; + sigmatch_table[DETECT_FILESHA1].RegisterTests = DetectFileSha1RegisterTests; + + SCLogDebug("registering filesha1 rule option"); + return; +} + +/** + * \brief this function is used to parse filesha1 options + * \brief into the current signature + * + * \param de_ctx pointer to the Detection Engine Context + * \param s pointer to the Current Signature + * \param str pointer to the user provided "filesha1" option + * + * \retval 0 on Success + * \retval -1 on Failure + */ +static int DetectFileSha1Setup (DetectEngineCtx *de_ctx, Signature *s, char *str) +{ + return DetectFileHashSetup(de_ctx, s, str, DETECT_FILESHA1); +} + +#ifdef UNITTESTS +static int SHA1MatchLookupString(ROHashTable *hash, char *string) +{ + uint8_t sha1[20]; + if (ReadHashString(sha1, string, "file", 88, 40) == 1) { + void *ptr = ROHashLookup(hash, &sha1, (uint16_t)sizeof(sha1)); + if (ptr == NULL) + return 0; + else + return 1; + } + return 0; +} + +static int SHA1MatchTest01(void) +{ + ROHashTable *hash = ROHashInit(4, 20); + if (hash == NULL) { + return 0; + } + if (LoadHashTable(hash, "447661c5de965bd4d837b50244467e37bddc184d", "file", 1, DETECT_FILESHA1) != 1) + return 0; + if (LoadHashTable(hash, "75a9af1e34dc0bb2f7fcde9d56b2503072ac35dd", "file", 2, DETECT_FILESHA1) != 1) + return 0; + if (LoadHashTable(hash, "53224a297bbb30631670fdcd2d295d87a1d328e9", "file", 3, DETECT_FILESHA1) != 1) + return 0; + if (LoadHashTable(hash, "3395856ce81f2b7382dee72602f798b642f14140", "file", 4, DETECT_FILESHA1) != 1) + return 0; + if (LoadHashTable(hash, "65559245709fe98052eb284577f1fd61c01ad20d", "file", 5, DETECT_FILESHA1) != 1) + return 0; + if (LoadHashTable(hash, "0931fd4e05e6ea81c75f8488ecc1db9e66f22cbb", "file", 6, DETECT_FILESHA1) != 1) + return 0; + + if (ROHashInitFinalize(hash) != 1) { + return 0; + } + + if (SHA1MatchLookupString(hash, "447661c5de965bd4d837b50244467e37bddc184d") != 1) + return 0; + if (SHA1MatchLookupString(hash, "75a9af1e34dc0bb2f7fcde9d56b2503072ac35dd") != 1) + return 0; + if (SHA1MatchLookupString(hash, "53224a297bbb30631670fdcd2d295d87a1d328e9") != 1) + return 0; + if (SHA1MatchLookupString(hash, "3395856ce81f2b7382dee72602f798b642f14140") != 1) + return 0; + if (SHA1MatchLookupString(hash, "65559245709fe98052eb284577f1fd61c01ad20d") != 1) + return 0; + if (SHA1MatchLookupString(hash, "0931fd4e05e6ea81c75f8488ecc1db9e66f22cbb") != 1) + return 0; + /* Shouldn't match */ + if (SHA1MatchLookupString(hash, "3333333333333333333333333333333333333333") == 1) + return 0; + + ROHashFree(hash); + return 1; +} +#endif + +void DetectFileSha1RegisterTests(void) +{ +#ifdef UNITTESTS + UtRegisterTest("SHA1MatchTest01", SHA1MatchTest01); +#endif +} + +#endif /* HAVE_NSS */ diff --git a/src/detect-filesha1.h b/src/detect-filesha1.h new file mode 100644 index 0000000000..d7504344c6 --- /dev/null +++ b/src/detect-filesha1.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * \author Duarte Silva + */ + +#ifndef __DETECT_FILESHA1_H__ +#define __DETECT_FILESHA1_H__ + +/* prototypes */ +void DetectFileSha1Register (void); + +#endif /* __DETECT_FILESHA1_H__ */ diff --git a/src/detect.c b/src/detect.c index c2ad76245a..767329b26d 100644 --- a/src/detect.c +++ b/src/detect.c @@ -108,6 +108,7 @@ #include "detect-filestore.h" #include "detect-filemagic.h" #include "detect-filemd5.h" +#include "detect-filesha1.h" #include "detect-filesize.h" #include "detect-dsize.h" #include "detect-flowvar.h" @@ -1861,6 +1862,14 @@ end: FileDisableMd5(pflow, STREAM_TOSERVER); } + /* see if this sgh requires us to consider file sha1 */ + if (!FileForceSha1() && (pflow->sgh_toserver == NULL || + !(pflow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILESHA1))) + { + SCLogDebug("disabling sha1 for flow"); + FileDisableSha1(pflow, STREAM_TOSERVER); + } + /* see if this sgh requires us to consider filesize */ if (pflow->sgh_toserver == NULL || !(pflow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) @@ -1892,6 +1901,14 @@ end: FileDisableMd5(pflow, STREAM_TOCLIENT); } + /* check if this flow needs sha1, if not disable it */ + if (!FileForceSha1() && (pflow->sgh_toclient == NULL || + !(pflow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILESHA1))) + { + SCLogDebug("disabling sha1 for flow"); + FileDisableSha1(pflow, STREAM_TOCLIENT); + } + /* see if this sgh requires us to consider filesize */ if (pflow->sgh_toclient == NULL || !(pflow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILESIZE)) @@ -2146,10 +2163,23 @@ int SignatureIsFilemagicInspecting(Signature *s) */ int SignatureIsFileMd5Inspecting(Signature *s) { - if (s == NULL) - return 0; + if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5)) + return 1; + + return 0; +} - if (s->file_flags & FILE_SIG_NEED_MD5) +/** + * \brief Check if a signature contains the filesha1 keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFileSha1Inspecting(Signature *s) +{ + if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1)) return 1; return 0; @@ -3996,7 +4026,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) SCLogDebug("sgh %p", sgh); SigGroupHeadSetFilemagicFlag(de_ctx, sgh); - SigGroupHeadSetFileMd5Flag(de_ctx, sgh); + SigGroupHeadSetFileHashFlag(de_ctx, sgh); SigGroupHeadSetFilesizeFlag(de_ctx, sgh); SigGroupHeadSetFilestoreCount(de_ctx, sgh); SCLogDebug("filestore count %u", sgh->filestore_cnt); @@ -4378,6 +4408,7 @@ void SigTableSetup(void) DetectFilestoreRegister(); DetectFilemagicRegister(); DetectFileMd5Register(); + DetectFileSha1Register(); DetectFilesizeRegister(); DetectAppLayerEventRegister(); DetectHttpUARegister(); diff --git a/src/detect.h b/src/detect.h index 3031f4d4c6..b3b947f5ed 100644 --- a/src/detect.h +++ b/src/detect.h @@ -937,11 +937,13 @@ typedef struct SigTableElmt_ { #define SIG_GROUP_HEAD_HAVEFILEMAGIC (1 << 20) #define SIG_GROUP_HEAD_HAVEFILEMD5 (1 << 21) #define SIG_GROUP_HEAD_HAVEFILESIZE (1 << 22) -#define SIG_GROUP_HEAD_MPM_DNSQUERY (1 << 23) -#define SIG_GROUP_HEAD_MPM_TLSSNI (1 << 24) -#define SIG_GROUP_HEAD_MPM_TLSISSUER (1 << 25) -#define SIG_GROUP_HEAD_MPM_TLSSUBJECT (1 << 26) -#define SIG_GROUP_HEAD_MPM_FD_SMTP (1 << 27) +#define SIG_GROUP_HEAD_HAVEFILESHA1 (1 << 23) + +#define SIG_GROUP_HEAD_MPM_DNSQUERY (1 << 24) +#define SIG_GROUP_HEAD_MPM_TLSSNI (1 << 25) +#define SIG_GROUP_HEAD_MPM_TLSISSUER (1 << 26) +#define SIG_GROUP_HEAD_MPM_TLSSUBJECT (1 << 27) +#define SIG_GROUP_HEAD_MPM_FD_SMTP (1 << 28) #define APP_MPMS_MAX 21 @@ -1278,6 +1280,7 @@ Signature *DetectGetTagSignature(void); int SignatureIsFilestoring(Signature *); int SignatureIsFilemagicInspecting(Signature *); int SignatureIsFileMd5Inspecting(Signature *); +int SignatureIsFileSha1Inspecting(Signature *s); int SignatureIsFilesizeInspecting(Signature *); int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int); diff --git a/src/flow.h b/src/flow.h index b5dbe57085..182f547c9f 100644 --- a/src/flow.h +++ b/src/flow.h @@ -66,40 +66,44 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOW_ACTION_DROP 0x00000200 /** Sgh for toserver direction set (even if it's NULL) */ -#define FLOW_SGH_TOSERVER 0x00000800 +#define FLOW_SGH_TOSERVER 0x00000400 /** Sgh for toclient direction set (even if it's NULL) */ -#define FLOW_SGH_TOCLIENT 0x00001000 +#define FLOW_SGH_TOCLIENT 0x00000800 /** packet to server direction has been logged in drop file (only in IPS mode) */ -#define FLOW_TOSERVER_DROP_LOGGED 0x00002000 +#define FLOW_TOSERVER_DROP_LOGGED 0x00001000 /** packet to client direction has been logged in drop file (only in IPS mode) */ -#define FLOW_TOCLIENT_DROP_LOGGED 0x00004000 +#define FLOW_TOCLIENT_DROP_LOGGED 0x00002000 /** alproto detect done. Right now we need it only for udp */ -#define FLOW_ALPROTO_DETECT_DONE 0x00008000 +#define FLOW_ALPROTO_DETECT_DONE 0x00004000 // vacany 1x /** Pattern matcher alproto detection done */ -#define FLOW_TS_PM_ALPROTO_DETECT_DONE 0x00020000 +#define FLOW_TS_PM_ALPROTO_DETECT_DONE 0x00008000 /** Probing parser alproto detection done */ -#define FLOW_TS_PP_ALPROTO_DETECT_DONE 0x00040000 +#define FLOW_TS_PP_ALPROTO_DETECT_DONE 0x00010000 /** Pattern matcher alproto detection done */ -#define FLOW_TC_PM_ALPROTO_DETECT_DONE 0x00100000 +#define FLOW_TC_PM_ALPROTO_DETECT_DONE 0x00020000 /** Probing parser alproto detection done */ -#define FLOW_TC_PP_ALPROTO_DETECT_DONE 0x00200000 -#define FLOW_TIMEOUT_REASSEMBLY_DONE 0x00800000 +#define FLOW_TC_PP_ALPROTO_DETECT_DONE 0x00040000 +#define FLOW_TIMEOUT_REASSEMBLY_DONE 0x00080000 /** even if the flow has files, don't store 'm */ -#define FLOW_FILE_NO_STORE_TS 0x01000000 -#define FLOW_FILE_NO_STORE_TC 0x02000000 +#define FLOW_FILE_NO_STORE_TS 0x00100000 +#define FLOW_FILE_NO_STORE_TC 0x00200000 /** flow is ipv4 */ -#define FLOW_IPV4 0x04000000 +#define FLOW_IPV4 0x00400000 /** flow is ipv6 */ -#define FLOW_IPV6 0x08000000 +#define FLOW_IPV6 0x00800000 /** no md5 on files in this flow */ -#define FLOW_FILE_NO_MD5_TS 0x10000000 -#define FLOW_FILE_NO_MD5_TC 0x20000000 +#define FLOW_FILE_NO_MD5_TS 0x01000000 +#define FLOW_FILE_NO_MD5_TC 0x02000000 + +/** no sha1 on files in this flow */ +#define FLOW_FILE_NO_SHA1_TS 0x04000000 +#define FLOW_FILE_NO_SHA1_TC 0x08000000 /** no size tracking of files in this flow */ #define FLOW_FILE_NO_SIZE_TS 0x40000000 diff --git a/src/log-file.c b/src/log-file.c index 1b594c5909..73cde22685 100644 --- a/src/log-file.c +++ b/src/log-file.c @@ -290,6 +290,14 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, const Packet *p, const } fprintf(fp, "\", "); } + if (ff->flags & FILE_SHA1) { + fprintf(fp, "\"sha1\": \""); + size_t x; + for (x = 0; x < sizeof(ff->sha1); x++) { + fprintf(fp, "%02x", ff->sha1[x]); + } + fprintf(fp, "\", "); + } #endif break; case FILE_STATE_TRUNCATED: @@ -437,7 +445,6 @@ static OutputCtx *LogFileLogInitCtx(ConfNode *conf) SCLogInfo("md5 calculation requires linking against libnss"); #endif } - FileForceTrackingEnable(); SCReturnPtr(output_ctx, "OutputCtx"); } diff --git a/src/log-filestore.c b/src/log-filestore.c index 99fa7e38d9..b87473ee0c 100644 --- a/src/log-filestore.c +++ b/src/log-filestore.c @@ -261,6 +261,14 @@ static void LogFilestoreLogCloseMetaFile(const File *ff) } fprintf(fp, "\n"); } + if (ff->flags & FILE_SHA1) { + fprintf(fp, "SHA1: "); + size_t x; + for (x = 0; x < sizeof(ff->sha1); x++) { + fprintf(fp, "%02x", ff->sha1[x]); + } + fprintf(fp, "\n"); + } #endif break; case FILE_STATE_TRUNCATED: diff --git a/src/output-json-file.c b/src/output-json-file.c index dfc034bdc7..76ad4c3db5 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -134,6 +134,15 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F } json_object_set_new(fjs, "md5", json_string(s)); } + if (ff->flags & FILE_SHA1) { + size_t x; + int i; + char s[256]; + for (i = 0, x = 0; x < sizeof(ff->sha1); x++) { + i += snprintf(&s[i], 255-i, "%02x", ff->sha1[x]); + } + json_object_set_new(fjs, "sha1", json_string(s)); + } #endif break; case FILE_STATE_TRUNCATED: diff --git a/src/suricata.c b/src/suricata.c index 3853d5bee0..66d94ff78b 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2332,7 +2332,7 @@ static int PostConfLoadedSetup(SCInstance *suri) #ifdef HAVE_NSS if (suri->run_mode != RUNMODE_CONF_TEST) { - /* init NSS for md5 */ + /* init NSS for hashing */ PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0); NSS_NoDB_Init(NULL); } diff --git a/src/util-error.c b/src/util-error.c index 94d3e02d05..8b6d48afe8 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -327,6 +327,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_NIC_OFFLOADING); CASE_CODE (SC_ERR_NO_FILES_FOR_PROTOCOL); CASE_CODE (SC_ERR_INVALID_HASH); + CASE_CODE (SC_ERR_NO_SHA1_SUPPORT); } return "UNKNOWN_ERROR"; diff --git a/src/util-error.h b/src/util-error.h index 63eee8ec3b..312ace1d7a 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -317,6 +317,7 @@ typedef enum { SC_ERR_NIC_OFFLOADING, SC_ERR_NO_FILES_FOR_PROTOCOL, SC_ERR_INVALID_HASH, + SC_ERR_NO_SHA1_SUPPORT, } SCError; const char *SCErrorToString(SCError); diff --git a/src/util-file.c b/src/util-file.c index 3fc7d5ff23..82efb90e67 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -51,6 +51,11 @@ static int g_file_force_magic = 0; */ static int g_file_force_md5 = 0; +/** \brief switch to force sha1 calculation on all files + * regardless of the rules. + */ +static int g_file_force_sha1 = 0; + /** \brief switch to force tracking off all files * regardless of the rules. */ @@ -74,6 +79,11 @@ void FileForceMd5Enable(void) g_file_force_md5 = 1; } +void FileForceSha1Enable(void) +{ + g_file_force_sha1 = 1; +} + int FileForceFilestore(void) { return g_file_force_filestore; @@ -89,6 +99,11 @@ int FileForceMd5(void) return g_file_force_md5; } +int FileForceSha1(void) +{ + return g_file_force_sha1; +} + void FileForceTrackingEnable(void) { g_file_force_tracking = 1; @@ -289,6 +304,8 @@ static void FileFree(File *ff) #ifdef HAVE_NSS if (ff->md5_ctx) HASH_Destroy(ff->md5_ctx); + if (ff->sha1_ctx) + HASH_Destroy(ff->sha1_ctx); #endif SCFree(ff); } @@ -363,6 +380,9 @@ static int AppendData(File *file, const uint8_t *data, uint32_t data_len) if (file->md5_ctx) { HASH_Update(file->md5_ctx, data, data_len); } + if (file->sha1_ctx) { + HASH_Update(file->sha1_ctx, data, data_len); + } #endif SCReturnInt(0); } @@ -396,11 +416,13 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len) if (FileStoreNoStoreCheck(ffc->tail) == 1) { #ifdef HAVE_NSS - /* no storage but forced md5 */ + /* no storage but forced hashing */ if (ffc->tail->md5_ctx) { - if (ffc->tail->md5_ctx) - HASH_Update(ffc->tail->md5_ctx, data, data_len); - + HASH_Update(ffc->tail->md5_ctx, data, data_len); + SCReturnInt(0); + } + if (ffc->tail->sha1_ctx) { + HASH_Update(ffc->tail->sha1_ctx, data, data_len); SCReturnInt(0); } #endif @@ -471,6 +493,10 @@ File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg, SCLogDebug("not doing md5 for this file"); ff->flags |= FILE_NOMD5; } + if (flags & FILE_NOSHA1) { + SCLogDebug("not doing sha1 for this file"); + ff->flags |= FILE_NOSHA1; + } if (flags & FILE_USE_DETECT) { SCLogDebug("considering content_inspect tracker when pruning"); ff->flags |= FILE_USE_DETECT; @@ -483,6 +509,12 @@ File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg, HASH_Begin(ff->md5_ctx); } } + if (!(ff->flags & FILE_NOSHA1) || g_file_force_sha1) { + ff->sha1_ctx = HASH_Create(HASH_AlgSHA1); + if (ff->sha1_ctx != NULL) { + HASH_Begin(ff->sha1_ctx); + } + } #endif ff->state = FILE_STATE_OPENED; @@ -517,9 +549,11 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data, if (data != NULL) { if (ff->flags & FILE_NOSTORE) { #ifdef HAVE_NSS - /* no storage but md5 */ + /* no storage but hashing */ if (ff->md5_ctx) HASH_Update(ff->md5_ctx, data, data_len); + if (ff->sha1_ctx) + HASH_Update(ff->sha1_ctx, data, data_len); #endif } else { if (AppendData(ff, data, data_len) != 0) { @@ -547,6 +581,11 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data, HASH_End(ff->md5_ctx, ff->md5, &len, sizeof(ff->md5)); ff->flags |= FILE_MD5; } + if (ff->sha1_ctx) { + unsigned int len = 0; + HASH_End(ff->sha1_ctx, ff->sha1, &len, sizeof(ff->sha1)); + ff->flags |= FILE_SHA1; + } #endif } @@ -682,6 +721,45 @@ void FileDisableMd5(Flow *f, uint8_t direction) SCReturn; } +/** + * \brief disable file sha1 calc for this flow + * + * \param f *LOCKED* flow + * \param direction flow direction +*/ +void FileDisableSha1(Flow *f, uint8_t direction) +{ + File *ptr = NULL; + + SCEnter(); + + DEBUG_ASSERT_FLOW_LOCKED(f); + + if (direction == STREAM_TOSERVER) + f->flags |= FLOW_FILE_NO_SHA1_TS; + else + f->flags |= FLOW_FILE_NO_SHA1_TC; + + FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + if (ffc != NULL) { + for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { + SCLogDebug("disabling sha1 for file %p from direction %s", + ptr, direction == STREAM_TOSERVER ? "toserver":"toclient"); + ptr->flags |= FILE_NOSHA1; + +#ifdef HAVE_NSS + /* destroy any ctx we may have so far */ + if (ptr->sha1_ctx != NULL) { + HASH_Destroy(ptr->sha1_ctx); + ptr->sha1_ctx = NULL; + } +#endif + } + } + + SCReturn; +} + /** * \brief disable file size tracking for this flow * @@ -731,7 +809,7 @@ void FileDisableStoringForFile(File *ff) ff->flags |= FILE_NOSTORE; if (ff->state == FILE_STATE_OPENED && FileSize(ff) >= (uint64_t)FileMagicSize()) { - if (g_file_force_md5 == 0 && g_file_force_tracking == 0) { + if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_tracking == 0) { (void)FileCloseFilePtr(ff, NULL, 0, (FILE_TRUNCATED|FILE_NOSTORE)); } diff --git a/src/util-file.h b/src/util-file.h index 96fee17a90..0c476bd9f4 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -189,6 +189,10 @@ void FileDisableMd5(Flow *f, uint8_t); void FileForceMd5Enable(void); int FileForceMd5(void); +void FileDisableSha1(Flow *f, uint8_t); +void FileForceSha1Enable(void); +int FileForceSha1(void); + void FileForceTrackingEnable(void); void FileStoreAllFiles(FileContainer *); diff --git a/src/util-lua-common.c b/src/util-lua-common.c index 367374d76c..4b712b9f62 100644 --- a/src/util-lua-common.c +++ b/src/util-lua-common.c @@ -593,8 +593,19 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil strlcat(md5, one, sizeof(md5)); } } + char sha1[41] = ""; + char *sha1ptr = sha1; + if (file->flags & FILE_SHA1) { + size_t x; + for (x = 0; x < sizeof(file->sha1); x++) { + char one[3] = ""; + snprintf(one, sizeof(one), "%02x", file->sha1[x]); + strlcat(sha1, one, sizeof(sha1)); + } + } #else char *md5ptr = NULL; + char *sha1ptr = NULL; #endif lua_pushnumber(luastate, file->file_id); @@ -603,6 +614,7 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil lua_pushnumber(luastate, FileSize(file)); lua_pushstring (luastate, file->magic); lua_pushstring(luastate, md5ptr); + lua_pushstring(luastate, sha1ptr); return 6; } -- 2.47.2