file: introduce per file 'track id'

Some protocols transfer multiple files in parallel. To support this add
a 'track id' to the API. This track id is set by the protocol parser. It
will use this id to indicate what file in the FileContainer it wants to
act on.

diff --git a/src/util-file.c b/src/util-file.c
index 447143cb8996f7f542f799f54d0961775d45c0bc..774ba6fcf58e696ea951a186d52374435bbea7ec 100644 (file)
--- a/src/util-file.c
+++ b/src/util-file.c
@@ -559,11 +559,10 @@ static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
     SCReturnInt(0);
 }
 
-/**
- *  \brief Store a chunk of file data in the flow. The open "flowfile"
- *         will be used.
+/** \internal
+ *  \brief Store/handle a chunk of file data in the File structure
  *
- *  \param ffc the container
+ *  \param ff the file
  *  \param data data chunk
  *  \param data_len data chunk len
  *
@@ -571,61 +570,115 @@ static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
  *  \retval -1 error
  *  \retval -2 no store for this file
  */
-int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
+static int FileAppendDataDo(File *ff, const uint8_t *data, uint32_t data_len)
 {
     SCEnter();
+#ifdef DEBUG_VALIDATION
+    BUG_ON(ff == NULL);
+#endif
 
-    if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
-        SCReturnInt(-1);
-    }
-
-    ffc->tail->size += data_len;
+    ff->size += data_len;
 
-    if (ffc->tail->state != FILE_STATE_OPENED) {
-        if (ffc->tail->flags & FILE_NOSTORE) {
+    if (ff->state != FILE_STATE_OPENED) {
+        if (ff->flags & FILE_NOSTORE) {
             SCReturnInt(-2);
         }
         SCReturnInt(-1);
     }
 
-    if (FileStoreNoStoreCheck(ffc->tail) == 1) {
+    if (FileStoreNoStoreCheck(ff) == 1) {
 #ifdef HAVE_NSS
         int hash_done = 0;
         /* no storage but forced hashing */
-        if (ffc->tail->md5_ctx) {
-            HASH_Update(ffc->tail->md5_ctx, data, data_len);
+        if (ff->md5_ctx) {
+            HASH_Update(ff->md5_ctx, data, data_len);
             hash_done = 1;
         }
-        if (ffc->tail->sha1_ctx) {
-            HASH_Update(ffc->tail->sha1_ctx, data, data_len);
+        if (ff->sha1_ctx) {
+            HASH_Update(ff->sha1_ctx, data, data_len);
             hash_done = 1;
         }
-        if (ffc->tail->sha256_ctx) {
-            HASH_Update(ffc->tail->sha256_ctx, data, data_len);
+        if (ff->sha256_ctx) {
+            HASH_Update(ff->sha256_ctx, data, data_len);
             hash_done = 1;
         }
 
         if (hash_done)
             SCReturnInt(0);
 #endif
-        if (g_file_force_tracking || (!(ffc->tail->flags & FILE_NOTRACK)))
+        if (g_file_force_tracking || (!(ff->flags & FILE_NOTRACK)))
             SCReturnInt(0);
 
-        ffc->tail->state = FILE_STATE_TRUNCATED;
+        ff->state = FILE_STATE_TRUNCATED;
         SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");
         SCReturnInt(-2);
     }
 
     SCLogDebug("appending %"PRIu32" bytes", data_len);
 
-    if (AppendData(ffc->tail, data, data_len) != 0) {
-        ffc->tail->state = FILE_STATE_ERROR;
+    if (AppendData(ff, data, data_len) != 0) {
+        ff->state = FILE_STATE_ERROR;
         SCReturnInt(-1);
     }
 
     SCReturnInt(0);
 }
 
+/**
+ *  \brief Store/handle a chunk of file data in the File structure
+ *         The last file in the FileContainer will be used.
+ *
+ *  \param ffc FileContainer used to append to
+ *  \param data data chunk
+ *  \param data_len data chunk len
+ *
+ *  \retval  0 ok
+ *  \retval -1 error
+ *  \retval -2 no store for this file
+ */
+int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
+{
+    SCEnter();
+
+    if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
+        SCReturnInt(-1);
+    }
+    int r = FileAppendDataDo(ffc->tail, data, data_len);
+    SCReturnInt(r);
+}
+
+/**
+ *  \brief Store/handle a chunk of file data in the File structure
+ *         The file with 'track_id' in the FileContainer will be used.
+ *
+ *  \param ffc FileContainer used to append to
+ *  \param track_id id to lookup the file
+ *  \param data data chunk
+ *  \param data_len data chunk len
+ *
+ *  \retval  0 ok
+ *  \retval -1 error
+ *  \retval -2 no store for this file
+ */
+int FileAppendDataById(FileContainer *ffc, uint32_t track_id,
+        const uint8_t *data, uint32_t data_len)
+{
+    SCEnter();
+
+    if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
+        SCReturnInt(-1);
+    }
+    File *ff = ffc->head;
+    for ( ; ff != NULL; ff = ff->next) {
+        if (track_id == ff->file_track_id) {
+            int r = FileAppendDataDo(ff, data, data_len);
+            SCReturnInt(r);
+        }
+    }
+    SCReturnInt(-1);
+}
+
+
 /**
  *  \brief Open a new File
  *
@@ -725,6 +778,18 @@ File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
 
     SCReturnPtr(ff, "File");
 }
+File *FileOpenFileWithId(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
+        uint32_t track_id, const uint8_t *name, uint16_t name_len,
+        const uint8_t *data, uint32_t data_len, uint16_t flags)
+{
+    File *ff = FileOpenFile(ffc, sbcfg, name, name_len, data, data_len, flags);
+    if (ff == NULL)
+        return NULL;
+
+    ff->file_track_id = track_id;
+    ff->flags |= FILE_USE_TRACKID;
+    return ff;
+}
 
 static int FileCloseFilePtr(File *ff, const uint8_t *data,
         uint32_t data_len, uint16_t flags)
@@ -819,6 +884,26 @@ int FileCloseFile(FileContainer *ffc, const uint8_t *data,
 
     SCReturnInt(0);
 }
+int FileCloseFileById(FileContainer *ffc, uint32_t track_id,
+        const uint8_t *data, uint32_t data_len, uint16_t flags)
+{
+    SCEnter();
+
+    if (ffc == NULL || ffc->tail == NULL) {
+        SCReturnInt(-1);
+    }
+
+    File *ff = ffc->head;
+    for ( ; ff != NULL; ff = ff->next) {
+        if (track_id == ff->file_track_id) {
+            if (FileCloseFilePtr(ff, data, data_len, flags) == -1) {
+                SCReturnInt(-1);
+            }
+            SCReturnInt(0);
+        }
+    }
+    SCReturnInt(-1);
+}
 
 /**
  *  \brief disable file storage for a flow

diff --git a/src/util-file.h b/src/util-file.h
index 82a7d56a7fa95e57bb586cc9bd2a9b3361162153..29bc967dd39643df645636b0e203a318667d793e 100644 (file)
--- a/src/util-file.h
+++ b/src/util-file.h
@@ -47,6 +47,7 @@
 #define FILE_STORED     BIT_U16(11)
 #define FILE_NOTRACK    BIT_U16(12) /**< track size of file */
 #define FILE_USE_DETECT BIT_U16(13) /**< use content_inspected tracker */
+#define FILE_USE_TRACKID    BIT_U16(14) /**< File::file_track_id field is in use */
 
 typedef enum FileState_ {
     FILE_STATE_NONE = 0,    /**< no state */
@@ -65,6 +66,8 @@ typedef struct File_ {
     int16_t state;
     StreamingBuffer *sb;
     uint64_t txid;                  /**< tx this file is part of */
+    uint32_t file_track_id;         /**< id used by protocol parser. Optional
+                                     *   only used if FILE_USE_TRACKID flag set */
     uint32_t file_store_id;         /**< id used in store file name file.<id> */
     uint8_t *name;
 #ifdef HAVE_MAGIC
@@ -120,6 +123,9 @@ void FileContainerAdd(FileContainer *, File *);
 File *FileOpenFile(FileContainer *, const StreamingBufferConfig *,
         const uint8_t *name, uint16_t name_len,
         const uint8_t *data, uint32_t data_len, uint16_t flags);
+File *FileOpenFileWithId(FileContainer *, const StreamingBufferConfig *,
+        uint32_t track_id, const uint8_t *name, uint16_t name_len,
+        const uint8_t *data, uint32_t data_len, uint16_t flags);
 
 /**
  *  \brief Close a File
@@ -134,6 +140,8 @@ File *FileOpenFile(FileContainer *, const StreamingBufferConfig *,
  */
 int FileCloseFile(FileContainer *, const uint8_t *data, uint32_t data_len,
         uint16_t flags);
+int FileCloseFileById(FileContainer *, uint32_t track_id,
+        const uint8_t *data, uint32_t data_len, uint16_t flags);
 
 /**
  *  \brief Store a chunk of file data in the flow. The open "flowfile"
@@ -147,6 +155,8 @@ int FileCloseFile(FileContainer *, const uint8_t *data, uint32_t data_len,
  *  \retval -1 error
  */
 int FileAppendData(FileContainer *, const uint8_t *data, uint32_t data_len);
+int FileAppendDataById(FileContainer *, uint32_t track_id,
+        const uint8_t *data, uint32_t data_len);
 
 /**
  *  \brief Tag a file for storing