From 84881bf1b892183c9567d7ad26737a047566ec05 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 2 May 2019 13:02:07 +0200 Subject: [PATCH] detect/file.magic: add sticky buffer Add sticky buffer to inspect file magic. Includes mpm support. --- src/detect-engine-register.h | 1 + src/detect-filemagic.c | 262 ++++++++++++++++++++++++++++++++--- src/detect-filemagic.h | 1 - 3 files changed, 245 insertions(+), 19 deletions(-) diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index b788d52ae3..644bb0681f 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -182,6 +182,7 @@ enum { DETECT_FILESTORE, DETECT_FILESTORE_POSTMATCH, DETECT_FILEMAGIC, + DETECT_FILE_MAGIC, DETECT_FILEMD5, DETECT_FILESHA1, DETECT_FILESHA256, diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index 0d13798292..9aefb60b28 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -32,7 +32,8 @@ #include "detect-engine.h" #include "detect-engine-mpm.h" -#include "detect-engine-state.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-content-inspection.h" #include "flow.h" #include "flow-var.h" @@ -47,6 +48,7 @@ #include "util-unittest-helper.h" #include "app-layer.h" +#include "app-layer-parser.h" #include "stream-tcp.h" @@ -83,6 +85,20 @@ static void DetectFilemagicRegisterTests(void); static void DetectFilemagicFree(void *); static int g_file_match_list_id = 0; +static int DetectFilemagicSetupSticky(DetectEngineCtx *de_ctx, Signature *s, const char *str); +static int g_file_magic_buffer_id = 0; + +static int PrefilterMpmFilemagicRegister(DetectEngineCtx *de_ctx, + SigGroupHead *sgh, MpmCtx *mpm_ctx, + const DetectMpmAppLayerRegistery *mpm_reg, int list_id); +static int DetectEngineInspectFilemagic( + DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, + const DetectEngineAppInspectionEngine *engine, + const Signature *s, + Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id); + +static int g_magic_thread_ctx_id = -1; + /** * \brief Registration function for keyword: filemagic */ @@ -96,9 +112,44 @@ void DetectFilemagicRegister(void) sigmatch_table[DETECT_FILEMAGIC].Free = DetectFilemagicFree; sigmatch_table[DETECT_FILEMAGIC].RegisterTests = DetectFilemagicRegisterTests; sigmatch_table[DETECT_FILEMAGIC].flags = SIGMATCH_QUOTES_MANDATORY|SIGMATCH_HANDLE_NEGATION; + sigmatch_table[DETECT_FILEMAGIC].alternative = DETECT_FILE_MAGIC; + + sigmatch_table[DETECT_FILE_MAGIC].name = "file.magic"; + sigmatch_table[DETECT_FILE_MAGIC].desc = "sticky buffer to match on the file magic"; + sigmatch_table[DETECT_FILE_MAGIC].url = DOC_URL DOC_VERSION "/rules/file-keywords.html#filemagic"; + sigmatch_table[DETECT_FILE_MAGIC].Setup = DetectFilemagicSetupSticky; + sigmatch_table[DETECT_FILE_MAGIC].flags = SIGMATCH_NOOPT|SIGMATCH_INFO_STICKY_BUFFER; g_file_match_list_id = DetectBufferTypeRegister("files"); + AppProto protos_ts[] = { + ALPROTO_HTTP, ALPROTO_SMTP, ALPROTO_FTP, ALPROTO_SMB, ALPROTO_NFS, 0 }; + AppProto protos_tc[] = { + ALPROTO_HTTP, ALPROTO_FTP, ALPROTO_SMB, ALPROTO_NFS, 0 }; + + for (int i = 0; protos_ts[i] != 0; i++) { + DetectAppLayerInspectEngineRegister2("file.magic", protos_ts[i], + SIG_FLAG_TOSERVER, 0, + DetectEngineInspectFilemagic, NULL); + + DetectAppLayerMpmRegister2("file.magic", SIG_FLAG_TOSERVER, 2, + PrefilterMpmFilemagicRegister, NULL, protos_ts[i], + 0); + } + for (int i = 0; protos_tc[i] != 0; i++) { + DetectAppLayerInspectEngineRegister2("file.magic", protos_tc[i], + SIG_FLAG_TOCLIENT, 0, + DetectEngineInspectFilemagic, NULL); + + DetectAppLayerMpmRegister2("file.magic", SIG_FLAG_TOCLIENT, 2, + PrefilterMpmFilemagicRegister, NULL, protos_tc[i], + 0); + } + + DetectBufferTypeSetDescriptionByName("file.magic", + "file magic"); + + g_file_magic_buffer_id = DetectBufferTypeGetByName("file.magic"); SCLogDebug("registering filemagic rule option"); return; } @@ -187,7 +238,8 @@ static int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, int ret = 0; DetectFilemagicData *filemagic = (DetectFilemagicData *)m; - DetectFilemagicThreadData *tfilemagic = (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, filemagic->thread_ctx_id); + DetectFilemagicThreadData *tfilemagic = + (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, g_magic_thread_ctx_id); if (tfilemagic == NULL) { SCReturnInt(0); } @@ -285,19 +337,16 @@ error: return NULL; } -static void *DetectFilemagicThreadInit(void *data) +static void *DetectFilemagicThreadInit(void *data /*@unused@*/) { const char *filename = NULL; FILE *fd = NULL; - DetectFilemagicData *filemagic = (DetectFilemagicData *)data; - BUG_ON(filemagic == NULL); - DetectFilemagicThreadData *t = SCMalloc(sizeof(DetectFilemagicThreadData)); + DetectFilemagicThreadData *t = SCCalloc(1, sizeof(DetectFilemagicThreadData)); if (unlikely(t == NULL)) { SCLogError(SC_ERR_MEM_ALLOC, "couldn't alloc ctx memory"); return NULL; } - memset(t, 0x00, sizeof(DetectFilemagicThreadData)); t->ctx = magic_open(0); if (t->ctx == NULL) { @@ -360,18 +409,19 @@ static void DetectFilemagicThreadFree(void *ctx) */ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str) { - DetectFilemagicData *filemagic = NULL; SigMatch *sm = NULL; - filemagic = DetectFilemagicParse(str, s->init_data->negated); + DetectFilemagicData *filemagic = DetectFilemagicParse(str, s->init_data->negated); if (filemagic == NULL) - goto error; - - filemagic->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic", - DetectFilemagicThreadInit, (void *)filemagic, - DetectFilemagicThreadFree, 1); - if (filemagic->thread_ctx_id == -1) - goto error; + return -1; + + if (g_magic_thread_ctx_id == -1) { + g_magic_thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic", + DetectFilemagicThreadInit, (void *)filemagic, + DetectFilemagicThreadFree, 1); + if (g_magic_thread_ctx_id == -1) + goto error; + } /* Okay so far so good, lets get this into a SigMatch * and put it in the Signature. */ @@ -388,8 +438,7 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, const ch return 0; error: - if (filemagic != NULL) - DetectFilemagicFree(filemagic); + DetectFilemagicFree(filemagic); if (sm != NULL) SCFree(sm); return -1; @@ -413,6 +462,183 @@ static void DetectFilemagicFree(void *ptr) } } +/* file.magic implementation */ + +/** + * \brief this function setup the file.magic keyword used in the rule + * + * \param de_ctx Pointer to the Detection Engine Context + * \param s Pointer to the Signature to which the current keyword belongs + * \param str Should hold an empty string always + * + * \retval 0 On success + */ +static int DetectFilemagicSetupSticky(DetectEngineCtx *de_ctx, Signature *s, const char *str) +{ + if (DetectBufferSetActiveList(s, g_file_magic_buffer_id) < 0) + return -1; + + if (g_magic_thread_ctx_id == -1) { + g_magic_thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic", + DetectFilemagicThreadInit, NULL, + DetectFilemagicThreadFree, 1); + if (g_magic_thread_ctx_id == -1) + return -1; + } + return 0; +} + +static InspectionBuffer *FilemagicGetDataCallback(DetectEngineThreadCtx *det_ctx, + const DetectEngineTransforms *transforms, + Flow *f, uint8_t flow_flags, File *cur_file, + int list_id, int local_file_id, bool first) +{ + SCEnter(); + + InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id); + InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, local_file_id); + if (buffer == NULL) + return NULL; + if (!first && buffer->inspect != NULL) + return buffer; + + if (cur_file->magic == NULL) { + DetectFilemagicThreadData *tfilemagic = + (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, g_magic_thread_ctx_id); + if (tfilemagic == NULL) { + return NULL; + } + + FilemagicThreadLookup(&tfilemagic->ctx, cur_file); + } + if (cur_file->magic == NULL) { + return NULL; + } + + const uint8_t *data = (const uint8_t *)cur_file->magic; + uint32_t data_len = (uint32_t)strlen(cur_file->magic); + + InspectionBufferSetup(buffer, data, data_len); + InspectionBufferApplyTransforms(buffer, transforms); + + SCReturnPtr(buffer, "InspectionBuffer"); +} + +static int DetectEngineInspectFilemagic( + DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, + const DetectEngineAppInspectionEngine *engine, + const Signature *s, + Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id) +{ + const DetectEngineTransforms *transforms = NULL; + if (!engine->mpm) { + transforms = engine->v2.transforms; + } + + FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, + f->alstate, flags); + if (ffc == NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + + int r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + int local_file_id = 0; + for (File *file = ffc->head; file != NULL; file = file->next) { + if (file->txid != tx_id) + continue; + + InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx, + transforms, f, flags, file, engine->sm_list, local_file_id, false); + if (buffer == NULL) + continue; + + det_ctx->buffer_offset = 0; + det_ctx->discontinue_matching = 0; + det_ctx->inspection_recursion_counter = 0; + int match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, + f, + (uint8_t *)buffer->inspect, + buffer->inspect_len, + buffer->inspect_offset, DETECT_CI_FLAGS_SINGLE, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL); + if (match == 1) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } else { + r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; + } + local_file_id++; + } + return r; +} + +typedef struct PrefilterMpmFilemagic { + int list_id; + const MpmCtx *mpm_ctx; + const DetectEngineTransforms *transforms; +} PrefilterMpmFilemagic; + +/** \brief Filedata Filedata Mpm prefilter callback + * + * \param det_ctx detection engine thread ctx + * \param p packet to inspect + * \param f flow to inspect + * \param txv tx to inspect + * \param pectx inspection context + */ +static void PrefilterTxFilemagic(DetectEngineThreadCtx *det_ctx, + const void *pectx, + Packet *p, Flow *f, void *txv, + const uint64_t idx, const uint8_t flags) +{ + SCEnter(); + + const PrefilterMpmFilemagic *ctx = (const PrefilterMpmFilemagic *)pectx; + const MpmCtx *mpm_ctx = ctx->mpm_ctx; + const int list_id = ctx->list_id; + + FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, + f->alstate, flags); + int local_file_id = 0; + if (ffc != NULL) { + File *file = ffc->head; + for (; file != NULL; file = file->next) { + if (file->txid != idx) + continue; + + InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx, + ctx->transforms, f, flags, file, list_id, local_file_id, true); + if (buffer == NULL) + continue; + + if (buffer->inspect_len >= mpm_ctx->minlen) { + (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx, + &det_ctx->mtcu, &det_ctx->pmq, + buffer->inspect, buffer->inspect_len); + } + } + } +} + +static void PrefilterMpmFilemagicFree(void *ptr) +{ + SCFree(ptr); +} + +static int PrefilterMpmFilemagicRegister(DetectEngineCtx *de_ctx, + SigGroupHead *sgh, MpmCtx *mpm_ctx, + const DetectMpmAppLayerRegistery *mpm_reg, int list_id) +{ + PrefilterMpmFilemagic *pectx = SCCalloc(1, sizeof(*pectx)); + if (pectx == NULL) + return -1; + pectx->list_id = list_id; + pectx->mpm_ctx = mpm_ctx; + pectx->transforms = &mpm_reg->v2.transforms; + + return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFilemagic, + mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress, + pectx, PrefilterMpmFilemagicFree, mpm_reg->pname); +} #ifdef UNITTESTS /* UNITTESTS */ /** diff --git a/src/detect-filemagic.h b/src/detect-filemagic.h index e36dfcbcfe..d7d93c3c58 100644 --- a/src/detect-filemagic.h +++ b/src/detect-filemagic.h @@ -32,7 +32,6 @@ typedef struct DetectFilemagicThreadData { } DetectFilemagicThreadData; typedef struct DetectFilemagicData { - int thread_ctx_id; uint8_t *name; /** name of the file to match */ BmCtx *bm_ctx; /** BM context */ uint16_t len; /** name length */ -- 2.47.2