From: Jason Ish <jason.ish@oisf.net>
Date: Mon, 7 Oct 2019 22:20:02 +0000 (-0600)
Subject: file extraction: always prune files after detect
X-Git-Tag: suricata-5.0.0~85
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ebcc4db84ac2c1957a6cc23b5154d7d6333f4cb8;p=thirdparty%2Fsuricata.git

file extraction: always prune files after detect

If a keyword like filemd5 was being used without a filestore,
or a file output enabled, it would be pruned before detection
had a chance to match.

Consolidate file pruning to the end of the flow worker so files
are available for detection even when a file output is not
enabled.

Redmine issue:
https://redmine.openinfosecfoundation.org/issues/2490
---

diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
index 6d29d5be1b..a172350ef6 100644
--- a/src/app-layer-ftp.c
+++ b/src/app-layer-ftp.c
@@ -1118,9 +1118,6 @@ static int FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
     }
 
 out:
-    if (ftpdata_state->files) {
-        FilePrune(ftpdata_state->files);
-    }
     return ret;
 }
 
diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 2f367daf7f..5529682b15 100644
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -146,7 +146,6 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
 
     FileSetTx(files->tail, txid);
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -301,7 +300,6 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -353,7 +351,6 @@ int HTPFileClose(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index e5578b6530..a660e6f5ff 100644
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -521,7 +521,6 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
 
     if (files != NULL) {
         SMTPPruneFiles(files);
-        FilePrune(files);
     }
 
     SCReturnInt(ret);
diff --git a/src/flow-worker.c b/src/flow-worker.c
index 7c6cf22401..91122f55df 100644
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -179,6 +179,18 @@ static TmEcode FlowWorkerThreadDeinit(ThreadVars *tv, void *data)
 TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq);
 TmEcode StreamTcp (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *);
 
+static void FlowPruneFiles(Packet *p)
+{
+    if (p->flow && p->flow->alstate) {
+        Flow *f = p->flow;
+        FileContainer *fc = AppLayerParserGetFiles(p->proto, f->alproto,
+            f->alstate, PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
+        if (fc != NULL) {
+            FilePrune(fc);
+        }
+    }
+}
+
 static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *preq, PacketQueue *unused)
 {
     FlowWorkerThreadData *fw = data;
@@ -281,6 +293,9 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *pr
     // Outputs.
     OutputLoggerLog(tv, p, fw->output_thread);
 
+    /* Prune any stored files. */
+    FlowPruneFiles(p);
+
     /*  Release tcp segments. Done here after alerting can use them. */
     if (p->flow != NULL && p->proto == IPPROTO_TCP) {
         FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
diff --git a/src/output-file.c b/src/output-file.c
index 22137bb30b..2747f2ed18 100644
--- a/src/output-file.c
+++ b/src/output-file.c
@@ -178,11 +178,6 @@ static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
     OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
     OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
 
-    if (ffc_ts && (p->flowflags & FLOW_PKT_TOSERVER))
-        FilePrune(ffc_ts);
-    if (ffc_tc && (p->flowflags & FLOW_PKT_TOCLIENT))
-        FilePrune(ffc_tc);
-
     return TM_ECODE_OK;
 }
 
diff --git a/src/output-filedata.c b/src/output-filedata.c
index acc1e6a980..9467365ca5 100644
--- a/src/output-filedata.c
+++ b/src/output-filedata.c
@@ -200,8 +200,6 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
                 }
             }
         }
-
-        FilePrune(ffc);
     }
 }