Adds options to mark when a file is final.

This takes the form of an option to add the pid of the process to file
names. Additionally, it adds a suffix to the file name to indicate it is
not finalized.

Adding the pid to the file name reduces the likelihood that a file is
overwritten when suricata is unexpectedly killed. The number in the
waldo file is only written out during a clean shutdown. In the event
of an improper shutdown, extracted files will be written using the old
number and existing files with the same name will be overwritten.

Writes extracted files and their metadata to a temporary file suffixed
with '.tmp'. Renames the files when they are completely done being
written. As-is there is no way to know that a file on disk is still
being written to by suricata.

diff --git a/doc/userguide/file-extraction/file-extraction.rst b/doc/userguide/file-extraction/file-extraction.rst
index b010531e79533231766c229d411aa892d3233744..a85567bf427d62e69acb244d882148e2ca705c72 100644 (file)
--- a/doc/userguide/file-extraction/file-extraction.rst
+++ b/doc/userguide/file-extraction/file-extraction.rst
@@ -44,8 +44,9 @@ drop dir must be configured.
       waldo: file.waldo # waldo file to store the file_id across runs
       max-open-files: 0 # how many files to keep open (O means none)
       write-meta: yes   # write a .meta file if set to yes
+      include-pid: yes  # include the pid in filenames if set to yes.
 
-Each file that is stored with have a name "file.<id>". The id will be reset and files will be overwritten unless the waldo option is used. A "file.<id>.meta" file is generated containing file metadata if write-meta is set to yes (default).
+Each file that is stored will have a name "file.<id>". The id will be reset and files will be overwritten unless the waldo option is used. A "file.<id>.meta" file is generated containing file metadata if write-meta is set to yes (default). If the include-pid option is set, the files will instead have a name "file.<pid>.<id>", and metafiles will be "file.<pid>.<id>.meta". Files will additionally have the suffix ".tmp" while they are open, which is only removed when they are finalized.
 
 
 ::

diff --git a/src/log-filestore.c b/src/log-filestore.c
index 8777b4a60fdc98733118b98efbe1bdefeff6bc2c..6b2c0f55e5afe56720e5b474462c891a612d7b68 100644 (file)
--- a/src/log-filestore.c
+++ b/src/log-filestore.c
@@ -63,6 +63,7 @@
 #define MODULE_NAME "LogFilestoreLog"
 
 static char g_logfile_base_dir[PATH_MAX] = "/tmp";
+static char g_working_file_suffix[PATH_MAX] = ".tmp";
 
 SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt);  /**< Atomic counter of simultaneously open files */
 
@@ -203,12 +204,25 @@ static uint32_t FileGetMaxOpenFiles(void)
     return g_file_store_max_open_files;
 }
 
-static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *filename, int ipver) {
+static int g_file_store_include_pid = 0;
+
+static void FileIncludePidEnable(void)
+{
+    g_file_store_include_pid = 1;
+}
+
+static int FileIncludePid(void)
+{
+    return g_file_store_include_pid;
+}
+
+static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *base_filename, int ipver) {
     if (!FileWriteMeta())
         return;
 
     char metafilename[PATH_MAX] = "";
-    snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
+    snprintf(metafilename, sizeof(metafilename), "%s.meta%s", base_filename,
+            g_working_file_suffix);
     FILE *fp = fopen(metafilename, "w+");
     if (fp != NULL) {
         char timebuf[64];
@@ -279,15 +293,19 @@ static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char
 
 static void LogFilestoreLogCloseMetaFile(const File *ff)
 {
-    if (!FileWriteMeta())
-        return;
-
-    char filename[PATH_MAX] = "";
-    snprintf(filename, sizeof(filename), "%s/file.%u",
-            g_logfile_base_dir, ff->file_store_id);
-    char metafilename[PATH_MAX] = "";
-    snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
-    FILE *fp = fopen(metafilename, "a");
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char final_filename[PATH_MAX] = "";
+    snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    char final_metafilename[PATH_MAX] = "";
+    snprintf(final_metafilename, sizeof(final_metafilename),
+            "%s.meta", final_filename);
+    char working_metafilename[PATH_MAX] = "";
+    snprintf(working_metafilename, sizeof(working_metafilename),
+            "%s%s", final_metafilename, g_working_file_suffix);
+    FILE *fp = fopen(working_metafilename, "a");
     if (fp != NULL) {
 #ifdef HAVE_MAGIC
         fprintf(fp, "MAGIC:             %s\n",
@@ -337,7 +355,39 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
 
         fclose(fp);
     } else {
-        SCLogInfo("opening %s failed: %s", metafilename, strerror(errno));
+        SCLogInfo("opening %s failed: %s", working_metafilename,
+                strerror(errno));
+    }
+}
+
+static void LogFilestoreFinalizeFiles(const File *ff) {
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char final_filename[PATH_MAX] = "";
+    snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    char working_filename[PATH_MAX] = "";
+    snprintf(working_filename, sizeof(working_filename), "%s%s",
+            final_filename, g_working_file_suffix);
+    if (rename(working_filename, final_filename) != 0) {
+        SCLogWarning(SC_WARN_RENAMING_FILE, "renaming file %s to %s failed",
+                working_filename, final_filename);
+        return;
+    }
+    if (FileWriteMeta()) {
+        LogFilestoreLogCloseMetaFile(ff);
+        char final_metafilename[PATH_MAX] = "";
+        snprintf(final_metafilename, sizeof(final_metafilename),
+                "%s.meta", final_filename);
+        char working_metafilename[PATH_MAX] = "";
+        snprintf(working_metafilename, sizeof(working_metafilename),
+                "%s%s", final_metafilename, g_working_file_suffix);
+        if (rename(working_metafilename, final_metafilename) != 0) {
+            SCLogWarning(SC_WARN_RENAMING_FILE,
+                    "renaming metafile %s to %s failed", working_metafilename,
+                    final_metafilename);
+        }
     }
 }
 
@@ -365,14 +415,20 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
 
     SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
 
-    snprintf(filename, sizeof(filename), "%s/file.%u",
-            g_logfile_base_dir, ff->file_store_id);
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char base_filename[PATH_MAX] = "";
+    snprintf(base_filename, sizeof(base_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    snprintf(filename, sizeof(filename), "%s%s", base_filename,
+            g_working_file_suffix);
 
     if (flags & OUTPUT_FILEDATA_FLAG_OPEN) {
         aft->file_cnt++;
 
         /* create a .meta file that contains time, src/dst/sp/dp/proto */
-        LogFilestoreLogCreateMetaFile(p, ff, filename, ipver);
+        LogFilestoreLogCreateMetaFile(p, ff, base_filename, ipver);
 
         if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
             SC_ATOMIC_ADD(filestore_open_file_cnt, 1);
@@ -425,7 +481,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
             ff->fd = -1;
             SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
         }
-        LogFilestoreLogCloseMetaFile(ff);
+        LogFilestoreFinalizeFiles(ff);
     }
 
     return 0;
@@ -598,6 +654,12 @@ static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf)
         }
     }
 
+    const char *include_pid = ConfNodeLookupChildValue(conf, "include-pid");
+    if (include_pid != NULL && ConfValIsTrue(include_pid)) {
+        FileIncludePidEnable();
+        SCLogInfo("enabling pid as a part of all file names");
+    }
+
     SCReturnPtr(output_ctx, "OutputCtx");
 }
 

diff --git a/src/util-error.c b/src/util-error.c
index 1e4075cd83e08d04a042eb65eb65f23d09f6c69c..15410359c9983373256537253e2b30190b5b5262 100644 (file)
--- a/src/util-error.c
+++ b/src/util-error.c
@@ -343,6 +343,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_NO_REDIS_ASYNC);
         CASE_CODE (SC_ERR_REDIS_CONFIG);
         CASE_CODE (SC_ERR_BYPASS_NOT_SUPPORTED);
+        CASE_CODE (SC_WARN_RENAMING_FILE);
     }
 
     return "UNKNOWN_ERROR";

diff --git a/src/util-error.h b/src/util-error.h
index cc73272a1fae19291edc52df305af4066d2bc044..12ce9aa19be1351ed197d25ada96e47703b6d895 100644 (file)
--- a/src/util-error.h
+++ b/src/util-error.h
@@ -332,7 +332,8 @@ typedef enum {
     SC_WARN_EVENT_DROPPED,
     SC_ERR_NO_REDIS_ASYNC,
     SC_ERR_REDIS_CONFIG,
-    SC_ERR_BYPASS_NOT_SUPPORTED
+    SC_ERR_BYPASS_NOT_SUPPORTED,
+    SC_WARN_RENAMING_FILE
 } SCError;
 
 const char *SCErrorToString(SCError);

diff --git a/suricata.yaml.in b/suricata.yaml.in
index 04055abf0521f5a2dc78de0eae2cd751b9f6e360..e0669e67554873bf6949765597bf9239e5f130ab 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -425,7 +425,11 @@ outputs:
   #
   # The files are stored to the log-dir in a format "file.<id>" where <id> is
   # an incrementing number starting at 1. For each file "file.<id>" a meta
-  # file "file.<id>.meta" is created.
+  # file "file.<id>.meta" is created. Before they are finalized, they will
+  # have a ".tmp" suffix to indicate that they are still being processed.
+  #
+  # If include-pid is yes, then the files are instead "file.<pid>.<id>", with
+  # meta files named as "file.<pid>.<id>.meta"
   #
   # File extraction depends on a lot of things to be fully done:
   # - file-store stream-depth. For optimal results, set this to 0 (unlimited)
@@ -449,6 +453,7 @@ outputs:
       # remain open for filestore by Suricata. Default value is 0 which
       # means files get closed after each write
       #max-open-files: 1000
+      include-pid: no # set to yes to include pid in file names
 
   # output module to log files tracked in a easily parsable json format
   - file-log: