From: Victor Julien Date: Wed, 10 Jan 2018 18:17:33 +0000 (+0100) Subject: detect: fix multiple files per tx inspect X-Git-Tag: suricata-4.1.0-beta1~316 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1df00749df6b250db79e9ad3760dea1a2b99cfb9;p=thirdparty%2Fsuricata.git detect: fix multiple files per tx inspect Fix the inspection of multiple files in a single TX, where new files may be added to the TX after inspection started. Assign the hard coded id DE_STATE_FLAG_FILE_INSPECT to the file inspect engine. Make sure that sigs that do file inspection and don't match on the current file always store a detailed state. This state will include the DE_STATE_FLAG_FILE_INSPECT flag. When the app-layer indicates a new file is available, for each sig that has the DE_STATE_FLAG_FILE_INSPECT flag set, reset part of the state so that the sig is evaluated again. --- diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index d420214676..321bbfd64d 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1180,13 +1180,14 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud, static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir) { + SCEnter(); if (tx && tx->de_state) { if (dir == STREAM_TOSERVER) { - SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW set"); - tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW; + SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set"); + tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW; } else if (STREAM_TOCLIENT) { - SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW set"); - tx->de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW; + SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set"); + tx->de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW; } } } diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 777a71911b..d88c809da9 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -390,12 +390,12 @@ static void SMTPPruneFiles(FileContainer *files) static void FlagDetectStateNewFile(SMTPTransaction *tx) { if (tx && tx->de_state) { - SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW set"); - tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW; + SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set"); + tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW; } else if (tx == NULL) { - SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW NOT set, no TX"); + SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW NOT set, no TX"); } else if (tx->de_state == NULL) { - SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW NOT set, no TX DESTATE"); + SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW NOT set, no TX DESTATE"); } } diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 9cc5ff2865..f47305951a 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -1176,8 +1176,8 @@ static int DeStateSigTest02(void) DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(IPPROTO_TCP, ALPROTO_HTTP, tx); FAIL_IF_NULL(tx_de_state); FAIL_IF(tx_de_state->dir_state[0].cnt != 1); - /* http_header(mpm): 6, uri: 4, method: 7, cookie: 8 */ - uint32_t expected_flags = (BIT_U32(6) | BIT_U32(4) | BIT_U32(7) |BIT_U32(8)); + /* http_header(mpm): 5, uri: 3, method: 6, cookie: 7 */ + uint32_t expected_flags = (BIT_U32(5) | BIT_U32(3) | BIT_U32(6) |BIT_U32(7)); FAIL_IF(tx_de_state->dir_state[0].head->store[0].flags != expected_flags); r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_HTTP, @@ -1609,6 +1609,375 @@ static int DeStateSigTest07(void) PASS; } +/** + * \test multiple files in a tx + */ +static int DeStateSigTest08(void) +{ + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 440\r\n" + "\r\n" + "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"AAAApicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "file"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "content\r\n" + "-----------------------------277531038314945\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"BBBBpicture2.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n" + "filecontent2\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + ThreadVars th_v; + TcpSession ssn; + + AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc(); + FAIL_IF_NULL(alp_tctx); + + memset(&th_v, 0, sizeof(th_v)); + memset(&ssn, 0, sizeof(ssn)); + + DetectEngineThreadCtx *det_ctx = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (content:\"POST\"; http_method; content:\"upload.cgi\"; http_uri; filename:\"BBBBpicture\"; filestore; sid:1; rev:1;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80); + FAIL_IF_NULL(f); + f->protoctx = &ssn; + f->proto = IPPROTO_TCP; + f->alproto = ALPROTO_HTTP; + + Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP); + FAIL_IF_NULL(p); + p->flow = f; + p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p->flowflags |= FLOW_PKT_TOSERVER; + p->flowflags |= FLOW_PKT_ESTABLISHED; + + StreamTcpInitConfig(TRUE); + + /* HTTP request with 1st part of the multipart body */ + + int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_START, httpbuf1, + httplen1); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf2, httplen2); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + HtpState *http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + File *file = files->head; + FAIL_IF_NULL(file); + FAIL_IF(file->flags & FILE_STORE); + + /* 2nd multipart body file */ + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf3, httplen3); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 1)); + + http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + file = files->head; + FAIL_IF_NULL(file); + FAIL_IF_NOT(file->flags & FILE_STORE); + + AppLayerParserThreadCtxFree(alp_tctx); + UTHFreeFlow(f); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + StreamTcpFreeConfig(TRUE); + PASS; +} + +/** + * \test multiple files in a tx. Both files should match + */ +static int DeStateSigTest09(void) +{ + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 440\r\n" + "\r\n" + "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "file"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "content\r\n" + "-----------------------------277531038314945\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n" + "filecontent2\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + ThreadVars th_v; + TcpSession ssn; + + AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc(); + FAIL_IF_NULL(alp_tctx); + + memset(&th_v, 0, sizeof(th_v)); + memset(&ssn, 0, sizeof(ssn)); + + DetectEngineThreadCtx *det_ctx = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (content:\"POST\"; http_method; content:\"upload.cgi\"; http_uri; filename:\"somepicture\"; filestore; sid:1; rev:1;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80); + FAIL_IF_NULL(f); + f->protoctx = &ssn; + f->proto = IPPROTO_TCP; + f->alproto = ALPROTO_HTTP; + + Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP); + FAIL_IF_NULL(p); + p->flow = f; + p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p->flowflags |= FLOW_PKT_TOSERVER; + p->flowflags |= FLOW_PKT_ESTABLISHED; + + StreamTcpInitConfig(TRUE); + + /* HTTP request with 1st part of the multipart body */ + + int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_START, httpbuf1, + httplen1); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf2, httplen2); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + HtpState *http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + File *file = files->head; + FAIL_IF_NULL(file); + FAIL_IF_NOT(file->flags & FILE_STORE); + + /* 2nd multipart body file */ + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf3, httplen3); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 1)); + + http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + file = files->head; + FAIL_IF_NULL(file); + FAIL_IF_NOT(file->flags & FILE_STORE); + + AppLayerParserThreadCtxFree(alp_tctx); + UTHFreeFlow(f); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + StreamTcpFreeConfig(TRUE); + PASS; +} + +/** + * \test multiple files in a tx. Both files should match. No other matches. + */ +static int DeStateSigTest10(void) +{ + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 440\r\n" + "\r\n" + "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "file"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "content\r\n" + "-----------------------------277531038314945\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n" + "filecontent2\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + ThreadVars th_v; + TcpSession ssn; + + AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc(); + FAIL_IF_NULL(alp_tctx); + + memset(&th_v, 0, sizeof(th_v)); + memset(&ssn, 0, sizeof(ssn)); + + DetectEngineThreadCtx *det_ctx = NULL; + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (filename:\"somepicture\"; filestore; sid:1; rev:1;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80); + FAIL_IF_NULL(f); + f->protoctx = &ssn; + f->proto = IPPROTO_TCP; + f->alproto = ALPROTO_HTTP; + + Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP); + FAIL_IF_NULL(p); + p->flow = f; + p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p->flowflags |= FLOW_PKT_TOSERVER; + p->flowflags |= FLOW_PKT_ESTABLISHED; + + StreamTcpInitConfig(TRUE); + + /* HTTP request with 1st part of the multipart body */ + + int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_START, httpbuf1, + httplen1); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf2, httplen2); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + HtpState *http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + File *file = files->head; + FAIL_IF_NULL(file); + FAIL_IF_NOT(file->flags & FILE_STORE); + + /* 2nd multipart body file */ + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER, httpbuf3, httplen3); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); + + r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP, + STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4); + FAIL_IF(r != 0); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 1)); + + http_state = f->alstate; + FAIL_IF_NULL(http_state); + FAIL_IF_NULL(http_state->files_ts); + + files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, + p->flow->alstate, STREAM_TOSERVER); + FAIL_IF_NULL(files); + file = files->head; + FAIL_IF_NULL(file); + FAIL_IF_NOT(file->flags & FILE_STORE); + + AppLayerParserThreadCtxFree(alp_tctx); + UTHFreeFlow(f); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + StreamTcpFreeConfig(TRUE); + PASS; +} + #endif void DeStateRegisterTests(void) @@ -1624,6 +1993,9 @@ void DeStateRegisterTests(void) UtRegisterTest("DeStateSigTest05", DeStateSigTest05); UtRegisterTest("DeStateSigTest06", DeStateSigTest06); UtRegisterTest("DeStateSigTest07", DeStateSigTest07); + UtRegisterTest("DeStateSigTest08", DeStateSigTest08); + UtRegisterTest("DeStateSigTest09", DeStateSigTest09); + UtRegisterTest("DeStateSigTest10", DeStateSigTest10); #endif return; diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h index 99631b77fc..db43f92b6b 100644 --- a/src/detect-engine-state.h +++ b/src/detect-engine-state.h @@ -52,20 +52,20 @@ /* per sig flags */ #define DE_STATE_FLAG_FULL_INSPECT BIT_U32(0) #define DE_STATE_FLAG_SIG_CANT_MATCH BIT_U32(1) - -#define DE_STATE_FLAG_FILE_TC_INSPECT BIT_U32(2) -#define DE_STATE_FLAG_FILE_TS_INSPECT BIT_U32(3) +/* flag set if file inspecting sig did not match, but might need to be + * re-evaluated for a new file in a tx */ +#define DE_STATE_ID_FILE_INSPECT 2UL +#define DE_STATE_FLAG_FILE_INSPECT BIT_U32(DE_STATE_ID_FILE_INSPECT) /* first bit position after the built-ins */ -#define DE_STATE_FLAG_BASE 4UL +#define DE_STATE_FLAG_BASE 3UL /* state flags * * Used by app-layer-parsers to notify us that new files * are available in the tx. */ -#define DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW BIT_U8(0) -#define DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW BIT_U8(1) +#define DETECT_ENGINE_STATE_FLAG_FILE_NEW BIT_U8(0) /* We have 2 possible state values to be used by ContinueDetection() while * trying to figure if we have fresh state to install or not. diff --git a/src/detect-engine.c b/src/detect-engine.c index b7d1385496..c162600dd5 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -195,6 +195,10 @@ static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int di SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id); } +/** + * \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT + * is assigned. + */ int DetectEngineAppInspectionEngine2Signature(Signature *s) { const int nlists = DetectBufferTypeMaxId(); @@ -205,6 +209,8 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s) SigMatchListSMBelongsTo(s, s->init_data->mpm_sm) : -1; + const int files_id = DetectBufferTypeGetByName("files"); + /* convert lists to SigMatchData arrays */ int i = 0; for (i = DETECT_SM_LIST_DYNAMIC_START; i < nlists; i++) { @@ -254,13 +260,23 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s) if (s->app_inspect == NULL) { s->app_inspect = new_engine; - last_id = new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */ + if (new_engine->sm_list == files_id) { + SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id); + new_engine->id = DE_STATE_ID_FILE_INSPECT; + } else { + new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */ + } /* prepend engine if forced or if our engine has a lower progress. */ } else if (prepend || (!head_is_mpm && s->app_inspect->progress > new_engine->progress)) { new_engine->next = s->app_inspect; s->app_inspect = new_engine; - new_engine->id = ++last_id; + if (new_engine->sm_list == files_id) { + SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id); + new_engine->id = DE_STATE_ID_FILE_INSPECT; + } else { + new_engine->id = ++last_id; + } } else { DetectEngineAppInspectionEngine *a = s->app_inspect; @@ -274,8 +290,14 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s) new_engine->next = a->next; a->next = new_engine; - new_engine->id = ++last_id; + if (new_engine->sm_list == files_id) { + SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id); + new_engine->id = DE_STATE_ID_FILE_INSPECT; + } else { + new_engine->id = ++last_id; + } } + SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id); s->flags |= SIG_FLAG_STATE_MATCH; diff --git a/src/detect.c b/src/detect.c index 50758ac355..fb450a93de 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1257,12 +1257,28 @@ static bool DetectRunTxInspectRule(ThreadVars *tv, TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags); if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) { if (file_no_match) { - DetectRunStoreStateTxFileOnly(scratch->sgh, f, tx->tx_ptr, tx->tx_id, - flow_flags, file_no_match); + /* if we have a mismatch on a file sig, we need to keep state. + * We may get another file on the same tx (for http and smtp + * at least), so for a new file we need to re-eval the sig. + * Thoughts / TODO: + * - not for some protos that have 1 file per tx (e.g. nfs) + * - maybe we only need this for file sigs that mix with + * other matches? E.g. 'POST + filename', is different than + * just 'filename'. + */ + DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s, + inspect_flags, flow_flags, file_no_match); } } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) { TRACE_SID_TXS(s->id, tx, "no need to store match sig, " "mpm won't trigger for it anymore"); + + if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) { + TRACE_SID_TXS(s->id, tx, "except that for new files, " + "we may have to revisit anyway"); + DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s, + inspect_flags, flow_flags, file_no_match); + } } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) { TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, " "mpm will revisit it"); @@ -1397,6 +1413,15 @@ static void DetectRunTx(ThreadVars *tv, if (tx.de_state != NULL) { const uint32_t old = array_idx; + /* if tx.de_state->flags has 'new file' set and sig below has + * 'file inspected' flag, reset the file part of the state */ + const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW); + if (have_new_file) { + SCLogDebug("%p/%"PRIu64" destate: need to consider new file", + tx.tx_ptr, tx_id); + tx.de_state->flags &= ~DETECT_ENGINE_STATE_FLAG_FILE_NEW; + } + SigIntId state_cnt = 0; DeStateStore *tx_store = tx.de_state->head; for (; tx_store != NULL; tx_store = tx_store->next) { @@ -1409,6 +1434,12 @@ static void DetectRunTx(ThreadVars *tv, { DeStateStoreItem *item = &tx_store->store[store_cnt]; SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags); + if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) { + /* remove part of the state. File inspect engine will now + * be able to run again */ + item->flags &= ~(DE_STATE_FLAG_SIG_CANT_MATCH|DE_STATE_FLAG_FULL_INSPECT|DE_STATE_FLAG_FILE_INSPECT); + SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags); + } det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid]; det_ctx->tx_candidates[array_idx].id = item->sid; det_ctx->tx_candidates[array_idx].flags = &item->flags;