Pull request #3183: file_api: file_data changes

Merge in SNORT/snort3 from ~VKAMBALA/snort3:file_info to master

Squashed commit of the following:

commit d8e4a5692a09e7394f410060dfb8017564421cac
Author: krishnakanth <vkambala@cisco.com>
Date:   Tue Nov 16 04:53:00 2021 -0500

    file_api: file_data changes

diff --git a/src/file_api/file_api.h b/src/file_api/file_api.h
index 27849e72993e64e5de4a1c9d1604107b182f6cf5..d7d1c87b7d65fa38077e9d31927beb7c6f490255 100644 (file)
--- a/src/file_api/file_api.h
+++ b/src/file_api/file_api.h
@@ -122,6 +122,18 @@ class FileInfo;
 class Flow;
 struct Packet;
 
+class UserFileDataBase
+{
+public:
+    UserFileDataBase() = default;
+    virtual ~UserFileDataBase() = default;
+    UserFileDataBase(const UserFileDataBase& other);
+    UserFileDataBase& operator=(const UserFileDataBase& other);
+
+private:
+    void copy(const UserFileDataBase& other);
+};
+
 class SO_PUBLIC FilePolicyBase
 {
 public:

diff --git a/src/file_api/file_cache.cc b/src/file_api/file_cache.cc
index e0f8a73b679c8a9d6753e94529b4aee17760cbca..293dc91041c8eeddacfb557409f1a1e9e3f8c74c 100644 (file)
--- a/src/file_api/file_cache.cc
+++ b/src/file_api/file_cache.cc
@@ -257,7 +257,25 @@ int FileCache::store_verdict(Flow* flow, FileInfo* file, int64_t timeout)
 
     FileContext* file_got = get_file(flow, file_id, true, timeout);
     if (file_got)
+    {
         *((FileInfo*)(file_got)) = *file;
+
+        if (FILE_VERDICT_PENDING == file->verdict and file != file_got)
+        {
+            if (file->get_file_data() and !file_got->get_file_data())
+            {
+                file_got->set_file_data(file->get_file_data());
+                file->set_file_data(nullptr);
+            }
+        }
+        else
+        {
+            if (file->get_file_data() and file != file_got)
+            {
+                file_got->set_file_data(nullptr);
+            }
+        }
+    }
     else
         return -1;
     return 0;

diff --git a/src/file_api/file_flows.cc b/src/file_api/file_flows.cc
index ac49d75b936e938bc69361715460cfdd4f63a364..e01bb21cae2f4309deb19a7d58bd8cad3ef15809 100644 (file)
--- a/src/file_api/file_flows.cc
+++ b/src/file_api/file_flows.cc
@@ -92,8 +92,8 @@ static void populate_trace_data(FileContext* context)
 
 void FileFlows::handle_retransmit(Packet* p)
 {
-    FILE_DEBUG(file_trace, DEFAULT_TRACE_OPTION_ID, TRACE_DEBUG_LEVEL, p ,
-       "handle_retransmit:queried for verdict\n");
+    FILE_DEBUG(file_trace, DEFAULT_TRACE_OPTION_ID, TRACE_DEBUG_LEVEL, p,
+        "handle_retransmit:queried for verdict\n");
     if (file_policy == nullptr)
         return;
 
@@ -105,8 +105,21 @@ void FileFlows::handle_retransmit(Packet* p)
         return;
     }
 
-    FileVerdict verdict = file_policy->signature_lookup(p, file);
+    FileContext* file_got = nullptr;
     FileCache* file_cache = FileService::get_file_cache();
+
+    if (!file->get_file_data())
+    {
+        if (file_cache)
+            file_got = file_cache->get_file(flow, pending_file_id, false);
+        if (file_got and file_got->get_file_data())
+        {
+            file->set_file_data(file_got->get_file_data());
+            file_got->set_file_data(nullptr);
+        }
+    }
+
+    FileVerdict verdict = file_policy->signature_lookup(p, file);
     if (file_cache)
     {
         FILE_DEBUG(file_trace, DEFAULT_TRACE_OPTION_ID, TRACE_DEBUG_LEVEL, p,
@@ -118,7 +131,6 @@ void FileFlows::handle_retransmit(Packet* p)
 
 FileFlows* FileFlows::get_file_flows(Flow* flow, bool to_create)
 {
-
     FileFlows* fd = (FileFlows*)flow->get_flow_data(FileFlows::file_flow_data_id);
 
     if (!to_create or fd)
@@ -152,8 +164,17 @@ void FileFlows::set_current_file_context(FileContext* ctx)
     // If we finished processing a file context object last time, delete it
     if (current_context_delete_pending and (current_context != ctx))
     {
+        int64_t file_id  = current_context->get_file_id();
         delete current_context;
         current_context_delete_pending = false;
+        FileCache* file_cache = FileService::get_file_cache();
+        assert(file_cache);
+        FileContext* file_got = file_cache->get_file(flow, file_id, false);
+        if (file_got and file_got->verdict == FILE_VERDICT_PENDING and current_context != file_got)
+        {
+            delete(file_got->get_file_data());
+            file_got->set_file_data(nullptr);
+        }
     }
     current_context = ctx;
     // Not using current_file_id so clear it
@@ -177,6 +198,22 @@ uint64_t FileFlows::get_new_file_instance()
 
 FileFlows::~FileFlows()
 {
+    FileCache* file_cache = FileService::get_file_cache();
+    assert(file_cache);
+    uint64_t file_id = 0;
+    if (current_context)
+        file_id = current_context->get_file_id();
+    else if (main_context)
+        file_id = main_context->get_file_id();
+
+    FileContext* file_got = file_cache->get_file(flow, file_id, false);
+
+    if (file_got and (file_got->verdict == FILE_VERDICT_PENDING))
+    {
+        delete (file_got->get_file_data());
+        file_got->set_file_data(nullptr);
+    }
+
     delete(main_context);
     if (current_context_delete_pending)
         delete(current_context);
@@ -230,7 +267,7 @@ FileContext* FileFlows::get_file_context(
     // First check if this file is currently being processed
     if (!multi_file_processing_id)
         multi_file_processing_id = file_id;
-    FileContext *context = get_partially_processed_context(multi_file_processing_id);
+    FileContext* context = get_partially_processed_context(multi_file_processing_id);
 
     // Otherwise check if it has been fully processed and is in the file cache. If the file is not
     // in the cache, don't add it.
@@ -345,15 +382,20 @@ bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_da
         context->set_file_id(file_id);
     }
 
-    if ( context->is_cacheable() and
-            (FileService::get_file_cache()->cached_verdict_lookup(p, context, file_policy) !=
-        FILE_VERDICT_UNKNOWN) )
+    if ( context->is_cacheable())
     {
-        context->processing_complete = true;
-        remove_processed_file_context(multi_file_processing_id);
-        if (PacketTracer::is_daq_activated())
-            populate_trace_data(context);
-        return false;
+        FileVerdict verdict = FileService::get_file_cache()->cached_verdict_lookup(p, context,
+            file_policy);
+        if (verdict != FILE_VERDICT_UNKNOWN and verdict != FILE_VERDICT_PENDING)
+        {
+            context->processing_complete = true;
+            remove_processed_file_context(multi_file_processing_id);
+            if (PacketTracer::is_daq_activated())
+                populate_trace_data(context);
+            return false;
+        }
+        else if (verdict == FILE_VERDICT_PENDING)
+            return true;
     }
 
     if (context->processing_complete and context->verdict != FILE_VERDICT_UNKNOWN)

diff --git a/src/file_api/file_lib.cc b/src/file_api/file_lib.cc
index 68395bb06161fa4c20839ba16539b980f4fb826c..3edd9cc99954a9aa877e0ee8d48456fa8948c33b 100644 (file)
--- a/src/file_api/file_lib.cc
+++ b/src/file_api/file_lib.cc
@@ -92,6 +92,12 @@ char* FileContext::get_UTF8_fname(size_t* converted_len)
 
 FileInfo::~FileInfo ()
 {
+    if (user_file_data)
+    {
+        delete user_file_data;
+        set_file_data(nullptr);
+    }
+
     if (sha256)
         delete[] sha256;
 }
@@ -286,6 +292,16 @@ int64_t FileInfo::get_max_file_capture_size()
     return (file_capture ? file_capture->get_max_file_capture_size() : 0);
 }
 
+void FileInfo::set_file_data(UserFileDataBase* fd)
+{
+    user_file_data = fd;
+}
+
+UserFileDataBase* FileInfo::get_file_data()
+{
+    return user_file_data;
+}
+
 FileContext::FileContext ()
 {
     file_type_context = nullptr;

diff --git a/src/file_api/file_lib.h b/src/file_api/file_lib.h
index c24b3ea466eb2239cdde7bcf4759f62f554be6bc..6a8cbebed2a232d60870183dfa8778fb9d51288c 100644 (file)
--- a/src/file_api/file_lib.h
+++ b/src/file_api/file_lib.h
@@ -79,7 +79,8 @@ public:
     bool is_file_capture_enabled();
     void set_policy_id(uint32_t id);
     uint32_t get_policy_id();
-
+    void set_file_data(UserFileDataBase* fd);
+    UserFileDataBase* get_file_data();
     // Preserve the file in memory until it is released
     // The file reserved will be returned and it will be detached from file context/session
     FileCaptureState reserve_file(FileCapture*& dest);
@@ -107,6 +108,7 @@ protected:
     bool file_capture_enabled = false;
     FileState file_state = { FILE_CAPTURE_SUCCESS, FILE_SIG_PROCESSING };
     uint32_t policy_id = 0;
+    UserFileDataBase* user_file_data = nullptr;
 
 private:
     void copy(const FileInfo& other);