filestore: track files getting stored per tx

Avoid evicting a tx before the filedata logger has decided it is
done.

(cherry picked from commit 56d3e28a3a122178270e81b73783e0f126486232)

diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs
index 500676d6c1d6558d7e1d2d9defcf21aee104d12a..f7db5792532564a540718f0c29468ad1ca03b41e 100644 (file)
--- a/rust/src/applayer.rs
+++ b/rust/src/applayer.rs
@@ -60,6 +60,7 @@ pub struct AppLayerTxData {
     /// track file open/logs so we can know how long to keep the tx
     pub files_opened: u32,
     pub files_logged: u32,
+    pub files_stored: u32,
 
     /// detection engine flags for use by detection engine
     detect_flags_ts: u64,
@@ -73,6 +74,7 @@ impl AppLayerTxData {
             logged: LoggerFlags::new(),
             files_opened: 0,
             files_logged: 0,
+            files_stored: 0,
             detect_flags_ts: 0,
             detect_flags_tc: 0,
         }

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 22727e60c12c88b5e8439092be6ec0fa6e093a03..2a41fd35b9ed088e7d8fd6e03b5a388d33ea49a2 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -880,6 +880,7 @@ FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
 
 extern int g_detect_disabled;
 extern bool g_file_logger_enabled;
+extern bool g_filedata_logger_enabled;
 
 /**
  * \brief remove obsolete (inspected and logged) transactions
@@ -1001,8 +1002,12 @@ void AppLayerParserTransactionsCleanup(Flow *f)
 
         /* if file logging is enabled, we keep a tx active while some of the files aren't
          * logged yet. */
-        if (txd && txd->files_opened && g_file_logger_enabled) {
-            if (txd->files_opened != txd->files_logged) {
+        if (txd && txd->files_opened) {
+            if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
+                skipped = true;
+                goto next;
+            }
+            if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
                 skipped = true;
                 goto next;
             }

diff --git a/src/output-filedata.c b/src/output-filedata.c
index 7a458737cc5d08b36dcada729487a86c0d265475..62b3bf5ed2bb8ea2038fcedca9dd2dc7cc9232c3 100644 (file)
--- a/src/output-filedata.c
+++ b/src/output-filedata.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2014 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -35,6 +35,8 @@
 #include "util-validate.h"
 #include "util-magic.h"
 
+bool g_filedata_logger_enabled = false;
+
 typedef struct OutputLoggerThreadStore_ {
     void *thread_data;
     struct OutputLoggerThreadStore_ *next;
@@ -97,6 +99,7 @@ int OutputRegisterFiledataLogger(LoggerId id, const char *name,
     }
 
     SCLogDebug("OutputRegisterFiledataLogger happy");
+    g_filedata_logger_enabled = true;
     return 0;
 }
 
@@ -129,6 +132,17 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
     return file_logged;
 }
 
+static void CloseFile(const Packet *p, Flow *f, File *file)
+{
+    void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
+    if (txv) {
+        AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
+        if (txd)
+            txd->files_stored++;
+    }
+    file->flags |= FILE_STORED;
+}
+
 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
         Packet *p, FileContainer *ffc, const uint8_t call_flags,
         const bool file_close, const bool file_trunc, const uint8_t dir)
@@ -162,7 +176,7 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
                     FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
                 }
                 CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
-                ff->flags |= FILE_STORED;
+                CloseFile(p, p->flow, ff);
                 continue;
             }
 
@@ -201,7 +215,7 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
 
                 /* all done */
                 if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
-                    ff->flags |= FILE_STORED;
+                    CloseFile(p, p->flow, ff);
                 }
             }
         }