detect: fix multiple files per tx inspect

Fix the inspection of multiple files in a single TX, where new files
may be added to the TX after inspection started.

Assign the hard coded id DE_STATE_FLAG_FILE_INSPECT to the file
inspect engine.

Make sure that sigs that do file inspection and don't match on the
current file always store a detailed state. This state will include
the DE_STATE_FLAG_FILE_INSPECT flag.

When the app-layer indicates a new file is available, for each sig
that has the DE_STATE_FLAG_FILE_INSPECT flag set, reset part of the
state so that the sig is evaluated again.

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index d420214676f85450cff0c5ba297296f03ba0e83f..321bbfd64d5c85b453b06fb76fc1bb0ebe1a2a2a 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1180,13 +1180,14 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
 
 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
 {
+    SCEnter();
     if (tx && tx->de_state) {
         if (dir == STREAM_TOSERVER) {
-            SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW set");
-            tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW;
+            SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
+            tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
         } else if (STREAM_TOCLIENT) {
-            SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW set");
-            tx->de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW;
+            SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
+            tx->de_state->dir_state[1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
         }
     }
 }

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 777a71911bfe93ee6d0b97b3db033fa1aebb8953..d88c809da9f404899517fd32871ddf25ed915c3e 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -390,12 +390,12 @@ static void SMTPPruneFiles(FileContainer *files)
 static void FlagDetectStateNewFile(SMTPTransaction *tx)
 {
     if (tx && tx->de_state) {
-        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW set");
-        tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW;
+        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW set");
+        tx->de_state->dir_state[0].flags |= DETECT_ENGINE_STATE_FLAG_FILE_NEW;
     } else if (tx == NULL) {
-        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW NOT set, no TX");
+        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW NOT set, no TX");
     } else if (tx->de_state == NULL) {
-        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW NOT set, no TX DESTATE");
+        SCLogDebug("DETECT_ENGINE_STATE_FLAG_FILE_NEW NOT set, no TX DESTATE");
     }
 }
 

diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c
index 9cc5ff2865cb075fce66d9d0e1d370c43c8500d6..f47305951aeec15704b2a811ba74a7a34207bad3 100644 (file)
--- a/src/detect-engine-state.c
+++ b/src/detect-engine-state.c
@@ -1176,8 +1176,8 @@ static int DeStateSigTest02(void)
     DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(IPPROTO_TCP, ALPROTO_HTTP, tx);
     FAIL_IF_NULL(tx_de_state);
     FAIL_IF(tx_de_state->dir_state[0].cnt != 1);
-    /* http_header(mpm): 6, uri: 4, method: 7, cookie: 8 */
-    uint32_t expected_flags = (BIT_U32(6) | BIT_U32(4) | BIT_U32(7) |BIT_U32(8));
+    /* http_header(mpm): 5, uri: 3, method: 6, cookie: 7 */
+    uint32_t expected_flags = (BIT_U32(5) | BIT_U32(3) | BIT_U32(6) |BIT_U32(7));
     FAIL_IF(tx_de_state->dir_state[0].head->store[0].flags != expected_flags);
 
     r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_HTTP,
@@ -1609,6 +1609,375 @@ static int DeStateSigTest07(void)
     PASS;
 }
 
+/**
+ * \test multiple files in a tx
+ */
+static int DeStateSigTest08(void)
+{
+    uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n"
+                         "Host: www.server.lan\r\n"
+                         "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n"
+                         "Content-Length: 440\r\n"
+                         "\r\n"
+                         "-----------------------------277531038314945\r\n"
+                         "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"AAAApicture1.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n";
+
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "file";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+    uint8_t httpbuf3[] = "content\r\n"
+                         "-----------------------------277531038314945\r\n";
+    uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
+
+    uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"BBBBpicture2.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "filecontent2\r\n"
+                         "-----------------------------277531038314945--";
+    uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
+
+    ThreadVars th_v;
+    TcpSession ssn;
+
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+    FAIL_IF_NULL(alp_tctx);
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&ssn, 0, sizeof(ssn));
+
+    DetectEngineThreadCtx *det_ctx = NULL;
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+    FAIL_IF_NULL(de_ctx);
+    de_ctx->flags |= DE_QUIET;
+
+    Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (content:\"POST\"; http_method; content:\"upload.cgi\"; http_uri; filename:\"BBBBpicture\"; filestore; sid:1; rev:1;)");
+    FAIL_IF_NULL(s);
+
+    SigGroupBuild(de_ctx);
+    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
+
+    Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
+    FAIL_IF_NULL(f);
+    f->protoctx = &ssn;
+    f->proto = IPPROTO_TCP;
+    f->alproto = ALPROTO_HTTP;
+
+    Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
+    FAIL_IF_NULL(p);
+    p->flow = f;
+    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    p->flowflags |= FLOW_PKT_TOSERVER;
+    p->flowflags |= FLOW_PKT_ESTABLISHED;
+
+    StreamTcpInitConfig(TRUE);
+
+    /* HTTP request with 1st part of the multipart body */
+
+    int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                                STREAM_TOSERVER | STREAM_START, httpbuf1,
+                                httplen1);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf2, httplen2);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    HtpState *http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    File *file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF(file->flags & FILE_STORE);
+
+    /* 2nd multipart body file */
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf3, httplen3);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 1));
+
+    http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF_NOT(file->flags & FILE_STORE);
+
+    AppLayerParserThreadCtxFree(alp_tctx);
+    UTHFreeFlow(f);
+    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
+    DetectEngineCtxFree(de_ctx);
+    StreamTcpFreeConfig(TRUE);
+    PASS;
+}
+
+/**
+ * \test multiple files in a tx. Both files should match
+ */
+static int DeStateSigTest09(void)
+{
+    uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n"
+                         "Host: www.server.lan\r\n"
+                         "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n"
+                         "Content-Length: 440\r\n"
+                         "\r\n"
+                         "-----------------------------277531038314945\r\n"
+                         "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n";
+
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "file";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+    uint8_t httpbuf3[] = "content\r\n"
+                         "-----------------------------277531038314945\r\n";
+    uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
+
+    uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "filecontent2\r\n"
+                         "-----------------------------277531038314945--";
+    uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
+
+    ThreadVars th_v;
+    TcpSession ssn;
+
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+    FAIL_IF_NULL(alp_tctx);
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&ssn, 0, sizeof(ssn));
+
+    DetectEngineThreadCtx *det_ctx = NULL;
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+    FAIL_IF_NULL(de_ctx);
+    de_ctx->flags |= DE_QUIET;
+
+    Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (content:\"POST\"; http_method; content:\"upload.cgi\"; http_uri; filename:\"somepicture\"; filestore; sid:1; rev:1;)");
+    FAIL_IF_NULL(s);
+
+    SigGroupBuild(de_ctx);
+    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
+
+    Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
+    FAIL_IF_NULL(f);
+    f->protoctx = &ssn;
+    f->proto = IPPROTO_TCP;
+    f->alproto = ALPROTO_HTTP;
+
+    Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
+    FAIL_IF_NULL(p);
+    p->flow = f;
+    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    p->flowflags |= FLOW_PKT_TOSERVER;
+    p->flowflags |= FLOW_PKT_ESTABLISHED;
+
+    StreamTcpInitConfig(TRUE);
+
+    /* HTTP request with 1st part of the multipart body */
+
+    int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                                STREAM_TOSERVER | STREAM_START, httpbuf1,
+                                httplen1);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf2, httplen2);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    HtpState *http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    File *file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF_NOT(file->flags & FILE_STORE);
+
+    /* 2nd multipart body file */
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf3, httplen3);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 1));
+
+    http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF_NOT(file->flags & FILE_STORE);
+
+    AppLayerParserThreadCtxFree(alp_tctx);
+    UTHFreeFlow(f);
+    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
+    DetectEngineCtxFree(de_ctx);
+    StreamTcpFreeConfig(TRUE);
+    PASS;
+}
+
+/**
+ * \test multiple files in a tx. Both files should match. No other matches.
+ */
+static int DeStateSigTest10(void)
+{
+    uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n"
+                         "Host: www.server.lan\r\n"
+                         "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n"
+                         "Content-Length: 440\r\n"
+                         "\r\n"
+                         "-----------------------------277531038314945\r\n"
+                         "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n";
+
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "file";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+    uint8_t httpbuf3[] = "content\r\n"
+                         "-----------------------------277531038314945\r\n";
+    uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
+
+    uint8_t httpbuf4[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "filecontent2\r\n"
+                         "-----------------------------277531038314945--";
+    uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
+
+    ThreadVars th_v;
+    TcpSession ssn;
+
+    AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+    FAIL_IF_NULL(alp_tctx);
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&ssn, 0, sizeof(ssn));
+
+    DetectEngineThreadCtx *det_ctx = NULL;
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+    FAIL_IF_NULL(de_ctx);
+    de_ctx->flags |= DE_QUIET;
+
+    Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any (filename:\"somepicture\"; filestore; sid:1; rev:1;)");
+    FAIL_IF_NULL(s);
+
+    SigGroupBuild(de_ctx);
+    DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
+
+    Flow *f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
+    FAIL_IF_NULL(f);
+    f->protoctx = &ssn;
+    f->proto = IPPROTO_TCP;
+    f->alproto = ALPROTO_HTTP;
+
+    Packet *p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
+    FAIL_IF_NULL(p);
+    p->flow = f;
+    p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
+    p->flowflags |= FLOW_PKT_TOSERVER;
+    p->flowflags |= FLOW_PKT_ESTABLISHED;
+
+    StreamTcpInitConfig(TRUE);
+
+    /* HTTP request with 1st part of the multipart body */
+
+    int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                                STREAM_TOSERVER | STREAM_START, httpbuf1,
+                                httplen1);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf2, httplen2);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    HtpState *http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    File *file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF_NOT(file->flags & FILE_STORE);
+
+    /* 2nd multipart body file */
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER, httpbuf3, httplen3);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF(PacketAlertCheck(p, 1));
+
+    r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
+                            STREAM_TOSERVER | STREAM_EOF, httpbuf4, httplen4);
+    FAIL_IF(r != 0);
+    SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 1));
+
+    http_state = f->alstate;
+    FAIL_IF_NULL(http_state);
+    FAIL_IF_NULL(http_state->files_ts);
+
+    files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto,
+                                                  p->flow->alstate, STREAM_TOSERVER);
+    FAIL_IF_NULL(files);
+    file = files->head;
+    FAIL_IF_NULL(file);
+    FAIL_IF_NOT(file->flags & FILE_STORE);
+
+    AppLayerParserThreadCtxFree(alp_tctx);
+    UTHFreeFlow(f);
+    DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
+    DetectEngineCtxFree(de_ctx);
+    StreamTcpFreeConfig(TRUE);
+    PASS;
+}
+
 #endif
 
 void DeStateRegisterTests(void)
@@ -1624,6 +1993,9 @@ void DeStateRegisterTests(void)
     UtRegisterTest("DeStateSigTest05", DeStateSigTest05);
     UtRegisterTest("DeStateSigTest06", DeStateSigTest06);
     UtRegisterTest("DeStateSigTest07", DeStateSigTest07);
+    UtRegisterTest("DeStateSigTest08", DeStateSigTest08);
+    UtRegisterTest("DeStateSigTest09", DeStateSigTest09);
+    UtRegisterTest("DeStateSigTest10", DeStateSigTest10);
 #endif
 
     return;

diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h
index 99631b77fc63f6e32467bcef007d19e714395052..db43f92b6b291a250041192728891dc092c87235 100644 (file)
--- a/src/detect-engine-state.h
+++ b/src/detect-engine-state.h
@@ -52,20 +52,20 @@
 /* per sig flags */
 #define DE_STATE_FLAG_FULL_INSPECT              BIT_U32(0)
 #define DE_STATE_FLAG_SIG_CANT_MATCH            BIT_U32(1)
-
-#define DE_STATE_FLAG_FILE_TC_INSPECT           BIT_U32(2)
-#define DE_STATE_FLAG_FILE_TS_INSPECT           BIT_U32(3)
+/* flag set if file inspecting sig did not match, but might need to be
+ * re-evaluated for a new file in a tx */
+#define DE_STATE_ID_FILE_INSPECT                2UL
+#define DE_STATE_FLAG_FILE_INSPECT              BIT_U32(DE_STATE_ID_FILE_INSPECT)
 
 /* first bit position after the built-ins */
-#define DE_STATE_FLAG_BASE                      4UL
+#define DE_STATE_FLAG_BASE                      3UL
 
 /* state flags
  *
  * Used by app-layer-parsers to notify us that new files
  * are available in the tx.
  */
-#define DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW    BIT_U8(0)
-#define DETECT_ENGINE_STATE_FLAG_FILE_TS_NEW    BIT_U8(1)
+#define DETECT_ENGINE_STATE_FLAG_FILE_NEW       BIT_U8(0)
 
 /* We have 2 possible state values to be used by ContinueDetection() while
  * trying to figure if we have fresh state to install or not.

diff --git a/src/detect-engine.c b/src/detect-engine.c
index b7d1385496a6aa5d3cab1a87d07cf3297e7de0bf..c162600dd58b5c1efe79eacd66b2b5ba101cd42c 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -195,6 +195,10 @@ static void AppendStreamInspectEngine(Signature *s, SigMatchData *stream, int di
     SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
 }
 
+/**
+ *  \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
+ *        is assigned.
+ */
 int DetectEngineAppInspectionEngine2Signature(Signature *s)
 {
     const int nlists = DetectBufferTypeMaxId();
@@ -205,6 +209,8 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
         SigMatchListSMBelongsTo(s, s->init_data->mpm_sm) :
         -1;
 
+    const int files_id = DetectBufferTypeGetByName("files");
+
     /* convert lists to SigMatchData arrays */
     int i = 0;
     for (i = DETECT_SM_LIST_DYNAMIC_START; i < nlists; i++) {
@@ -254,13 +260,23 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
 
         if (s->app_inspect == NULL) {
             s->app_inspect = new_engine;
-            last_id = new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
+            if (new_engine->sm_list == files_id) {
+                SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
+                new_engine->id = DE_STATE_ID_FILE_INSPECT;
+            } else {
+                new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
+            }
 
         /* prepend engine if forced or if our engine has a lower progress. */
         } else if (prepend || (!head_is_mpm && s->app_inspect->progress > new_engine->progress)) {
             new_engine->next = s->app_inspect;
             s->app_inspect = new_engine;
-            new_engine->id = ++last_id;
+            if (new_engine->sm_list == files_id) {
+                SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
+                new_engine->id = DE_STATE_ID_FILE_INSPECT;
+            } else {
+                new_engine->id = ++last_id;
+            }
 
         } else {
             DetectEngineAppInspectionEngine *a = s->app_inspect;
@@ -274,8 +290,14 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
 
             new_engine->next = a->next;
             a->next = new_engine;
-            new_engine->id = ++last_id;
+            if (new_engine->sm_list == files_id) {
+                SCLogDebug("sid %u: engine %p/%u is FILE ENGINE", s->id, new_engine, new_engine->id);
+                new_engine->id = DE_STATE_ID_FILE_INSPECT;
+            } else {
+                new_engine->id = ++last_id;
+            }
         }
+
         SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
 
         s->flags |= SIG_FLAG_STATE_MATCH;

diff --git a/src/detect.c b/src/detect.c
index 50758ac35521e8f1d4df6218474f8a4a1dfd4417..fb450a93dee5feb3fc1a6df3370da59c0255384e 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1257,12 +1257,28 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
         TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags);
         if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) {
             if (file_no_match) {
-                DetectRunStoreStateTxFileOnly(scratch->sgh, f, tx->tx_ptr, tx->tx_id,
-                        flow_flags, file_no_match);
+                /* if we have a mismatch on a file sig, we need to keep state.
+                 * We may get another file on the same tx (for http and smtp
+                 * at least), so for a new file we need to re-eval the sig.
+                 * Thoughts / TODO:
+                 *  - not for some protos that have 1 file per tx (e.g. nfs)
+                 *  - maybe we only need this for file sigs that mix with
+                 *    other matches? E.g. 'POST + filename', is different than
+                 *    just 'filename'.
+                 */
+                DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
+                        inspect_flags, flow_flags, file_no_match);
             }
         } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) {
             TRACE_SID_TXS(s->id, tx, "no need to store match sig, "
                     "mpm won't trigger for it anymore");
+
+            if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) {
+                TRACE_SID_TXS(s->id, tx, "except that for new files, "
+                        "we may have to revisit anyway");
+                DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
+                        inspect_flags, flow_flags, file_no_match);
+            }
         } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
             TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
                     "mpm will revisit it");
@@ -1397,6 +1413,15 @@ static void DetectRunTx(ThreadVars *tv,
         if (tx.de_state != NULL) {
             const uint32_t old = array_idx;
 
+            /* if tx.de_state->flags has 'new file' set and sig below has
+             * 'file inspected' flag, reset the file part of the state */
+            const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW);
+            if (have_new_file) {
+                SCLogDebug("%p/%"PRIu64" destate: need to consider new file",
+                        tx.tx_ptr, tx_id);
+                tx.de_state->flags &= ~DETECT_ENGINE_STATE_FLAG_FILE_NEW;
+            }
+
             SigIntId state_cnt = 0;
             DeStateStore *tx_store = tx.de_state->head;
             for (; tx_store != NULL; tx_store = tx_store->next) {
@@ -1409,6 +1434,12 @@ static void DetectRunTx(ThreadVars *tv,
                 {
                     DeStateStoreItem *item = &tx_store->store[store_cnt];
                     SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags);
+                    if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) {
+                        /* remove part of the state. File inspect engine will now
+                         * be able to run again */
+                        item->flags &= ~(DE_STATE_FLAG_SIG_CANT_MATCH|DE_STATE_FLAG_FULL_INSPECT|DE_STATE_FLAG_FILE_INSPECT);
+                        SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags);
+                    }
                     det_ctx->tx_candidates[array_idx].s = de_ctx->sig_array[item->sid];
                     det_ctx->tx_candidates[array_idx].id = item->sid;
                     det_ctx->tx_candidates[array_idx].flags = &item->flags;