file extraction: always prune files after detect

If a keyword like filemd5 was being used without a filestore,
or a file output enabled, it would be pruned before detection
had a chance to match.

Consolidate file pruning to the end of the flow worker so files
are available for detection even when a file output is not
enabled.

Redmine issue:
https://redmine.openinfosecfoundation.org/issues/2490

diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
index 6d29d5be1bdeb08927a402f9ca9c127d3520c639..a172350ef6297db83bedb56b41c1bdf4212bacd4 100644 (file)
--- a/src/app-layer-ftp.c
+++ b/src/app-layer-ftp.c
@@ -1118,9 +1118,6 @@ static int FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
     }
 
 out:
-    if (ftpdata_state->files) {
-        FilePrune(ftpdata_state->files);
-    }
     return ret;
 }
 

diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 2f367daf7fbea37868547258688cc385ab0a234e..5529682b152250900ba590556e219ede5c469d6a 100644 (file)
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -146,7 +146,6 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
 
     FileSetTx(files->tail, txid);
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -301,7 +300,6 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -353,7 +351,6 @@ int HTPFileClose(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index e5578b6530d144dc0d0733e2363391ca4fc11ed2..a660e6f5fff5f106e59662cf17c51d15fba102a6 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -521,7 +521,6 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
 
     if (files != NULL) {
         SMTPPruneFiles(files);
-        FilePrune(files);
     }
 
     SCReturnInt(ret);

diff --git a/src/flow-worker.c b/src/flow-worker.c
index 7c6cf224013360378cac73ef045d9dd6cd20258e..91122f55dfa1e2253720c862a0a0b96ac09f4fc3 100644 (file)
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -179,6 +179,18 @@ static TmEcode FlowWorkerThreadDeinit(ThreadVars *tv, void *data)
 TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq);
 TmEcode StreamTcp (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *);
 
+static void FlowPruneFiles(Packet *p)
+{
+    if (p->flow && p->flow->alstate) {
+        Flow *f = p->flow;
+        FileContainer *fc = AppLayerParserGetFiles(p->proto, f->alproto,
+            f->alstate, PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
+        if (fc != NULL) {
+            FilePrune(fc);
+        }
+    }
+}
+
 static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *preq, PacketQueue *unused)
 {
     FlowWorkerThreadData *fw = data;
@@ -281,6 +293,9 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *pr
     // Outputs.
     OutputLoggerLog(tv, p, fw->output_thread);
 
+    /* Prune any stored files. */
+    FlowPruneFiles(p);
+
     /*  Release tcp segments. Done here after alerting can use them. */
     if (p->flow != NULL && p->proto == IPPROTO_TCP) {
         FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);

diff --git a/src/output-file.c b/src/output-file.c
index 22137bb30b90c6dd6899a04b6f67915ab92539f7..2747f2ed187856291b316e2ac02126775e4854d3 100644 (file)
--- a/src/output-file.c
+++ b/src/output-file.c
@@ -178,11 +178,6 @@ static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
     OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
     OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
 
-    if (ffc_ts && (p->flowflags & FLOW_PKT_TOSERVER))
-        FilePrune(ffc_ts);
-    if (ffc_tc && (p->flowflags & FLOW_PKT_TOCLIENT))
-        FilePrune(ffc_tc);
-
     return TM_ECODE_OK;
 }
 

diff --git a/src/output-filedata.c b/src/output-filedata.c
index acc1e6a980cf81d193eea3bb0b60f02b6847e35a..9467365ca5e3876699bb2ac7b2058bc9abc51b45 100644 (file)
--- a/src/output-filedata.c
+++ b/src/output-filedata.c
@@ -200,8 +200,6 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
                 }
             }
         }
-
-        FilePrune(ffc);
     }
 }