Pull request #4604: file_api, http_inspect: add info about partial download to FileInfo

Merge in SNORT/snort3 from ~OTORUBAR/snort3:partial_download to master

Squashed commit of the following:

commit db3bf9c938d28a59851e9a74969cb580a78fcf78
Author: otorubar <otorubar@cisco.com>
Date:   Fri Jan 31 10:20:46 2025 -0800

    file_api, http_inspect: add info about partial download to FileInfo

diff --git a/src/file_api/file_flows.cc b/src/file_api/file_flows.cc
index eb3249645d22a623edb77483daeeb3b8decfad1f..3252d7b6178cb20a6bc11c7c27dab52159c9e987 100644 (file)
--- a/src/file_api/file_flows.cc
+++ b/src/file_api/file_flows.cc
@@ -326,7 +326,7 @@ void FileFlows::remove_processed_file_context(uint64_t file_id)
 bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_data,
     int data_size, uint64_t offset, FileDirection dir, uint64_t multi_file_processing_id,
     FilePosition position, const uint8_t* fname, uint32_t name_size,
-    const uint8_t* url, uint32_t url_size, const std::string& host_name)
+    const uint8_t* url, uint32_t url_size, const std::string& host_name, const bool is_partial)
 {
     int64_t file_depth = FileService::get_max_file_depth();
     bool continue_processing;
@@ -351,9 +351,10 @@ bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_da
         return false;
     }
 
-    if (context->has_to_re_eval() and context->processing_complete)
+    if (context->has_to_re_eval() and context->processing_complete and not is_partial)
         context->reset();
 
+    context->set_partial_flag(is_partial);
     context->set_weak_file_name((const char*)fname, name_size);
     context->set_weak_url((const char*)url, url_size);
     context->set_host(host_name.c_str(), host_name.size());
@@ -378,7 +379,7 @@ bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_da
         context->set_file_id(file_id);
     }
 
-    if (context->is_cacheable() and not is_new_context)
+    if ((context->is_cacheable() and not is_new_context) or context->is_partial_download()) // If partial header was seen - check file_cache
     {
         FileVerdict verdict = FileService::get_file_cache()->cached_verdict_lookup(p, context,
             file_policy);

diff --git a/src/file_api/file_flows.h b/src/file_api/file_flows.h
index 1a0686ec5cee8f16ca4a8c733154e7e02d4a602d..c2f8efb37988032a163d3afec58a327f9add3195 100644 (file)
--- a/src/file_api/file_flows.h
+++ b/src/file_api/file_flows.h
@@ -97,7 +97,7 @@ public:
     bool file_process(Packet* p, uint64_t file_id, const uint8_t* file_data,
         int data_size, uint64_t offset, FileDirection, uint64_t multi_file_processing_id=0,
         FilePosition=SNORT_FILE_POSITION_UNKNOWN, const uint8_t* fname = nullptr, uint32_t name_size = 0,
-        const uint8_t* url = nullptr, uint32_t url_size = 0, const std::string& host_name = "");
+        const uint8_t* url = nullptr, uint32_t url_size = 0, const std::string& host_name = "", const bool is_partial = false);
 
     static unsigned file_flow_data_id;
 

diff --git a/src/file_api/file_lib.cc b/src/file_api/file_lib.cc
index 09aa3a24d5b6df335342d594065dccf80a1ab6d6..5fb9dd2b3060ef50b13cc84f91dabd9c6cd1f3ed 100644 (file)
--- a/src/file_api/file_lib.cc
+++ b/src/file_api/file_lib.cc
@@ -137,6 +137,7 @@ void FileInfo::copy(const FileInfo& other, bool clear_data)
     host_set = other.host_set;
     verdict = other.verdict;
     file_type_enabled = other.file_type_enabled;
+    is_partial = other.is_partial;
     file_signature_enabled = other.file_signature_enabled;
     file_capture_enabled = other.file_capture_enabled;
     file_state = other.file_state;
@@ -510,6 +511,16 @@ void FileContext::check_policy(Flow* flow, FileDirection dir, FilePolicyBase* po
     policy->policy_check(flow, this);
 }
 
+void FileInfo::set_partial_flag(bool partial)
+{
+    is_partial = partial;
+}
+
+bool FileInfo::is_partial_download() const
+{
+    return is_partial;
+}
+
 void FileInfo::reset()
 {
     verdict = FILE_VERDICT_UNKNOWN;
@@ -1092,5 +1103,15 @@ TEST_CASE ("re_eval", "[file_info]")
     info.unset_re_eval();
     CHECK (false == info.has_to_re_eval());
 }
+
+TEST_CASE ("is_partial", "[file_info]")
+{
+    FI_TEST info;
+    CHECK (false == info.is_partial_download());
+    info.set_partial_flag(true);
+    CHECK (true == info.is_partial_download());
+    info.set_partial_flag(false);
+    CHECK (false == info.is_partial_download());
+}
 #endif
 

diff --git a/src/file_api/file_lib.h b/src/file_api/file_lib.h
index ef1d7f7770f59798476ac984c0929bca1c617967..014e888733511e7c6f1e070722e13d585c5a5ef5 100644 (file)
--- a/src/file_api/file_lib.h
+++ b/src/file_api/file_lib.h
@@ -111,9 +111,8 @@ public:
     bool has_to_re_eval();
     void unset_re_eval();
 
-    // Flag which indicates that the file requires re-eval even though it was fully processed before.
-    // If "true" and the policy was checked - has to be set to "false" again.
-    bool re_eval = false;
+    void set_partial_flag(bool partial);
+    bool is_partial_download() const;
 
 protected:
     std::string file_name;
@@ -134,6 +133,12 @@ protected:
     FileState file_state = { FILE_CAPTURE_SUCCESS, FILE_SIG_PROCESSING };
     uint32_t policy_id = 0;
     UserFileDataBase* user_file_data = nullptr;
+
+    // Flag which indicates that the file requires re-eval even though it was fully processed before.
+    // If "true" and the policy was checked - has to be set to "false" again.
+    bool re_eval = false;
+    // Indicates that file transmission goes through 206 HTTP Partial Content
+    bool is_partial = false;
 };
 
 class SO_PUBLIC FileContext : public FileInfo
@@ -176,7 +181,6 @@ public:
     // Configuration functions
     void remove_segments();
     void reset();
-    
 private:
     uint64_t processed_bytes = 0;
     void* file_type_context;

diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index 6ed1f03b8dcdc3151488897ef20af19b1c16173a..cd13c5d74fb0b594ef44839a826a538656c39d1a 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -603,6 +603,7 @@ static FilePosition find_range_file_pos(const std::string& hdr_content, bool fro
     if (errno or end_ptr != hdr_content.c_str() + dash_pos)
         return SNORT_FILE_POSITION_UNKNOWN;
 
+    // return middle no matter what range actually is
     if (range_start != 0)
         return SNORT_FILE_MIDDLE;
 
@@ -652,6 +653,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
     const bool front = (body_octets == 0) &&
         (session_data->partial_inspected_octets[source_id] == 0);
     const bool back = (session_data->cutter[source_id] == nullptr) || tcp_close;
+    bool is_partially_downloaded = false;
 
     if (session_data->status_code_num != 206 or source_id != SRC_SERVER)
     {
@@ -680,6 +682,8 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
 
         if (file_position == SNORT_FILE_POSITION_UNKNOWN)
             return;
+
+        is_partially_downloaded = true;
     }
 
     const int32_t fp_length = (file_data.length() <=
@@ -695,7 +699,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
     uint64_t file_index = get_header(source_id)->get_file_cache_index();
     // Get host from the header field.
     std::string host = get_header(source_id)->get_host_header_field();
-    
+
     const uint8_t* filename_buffer = nullptr;
     uint32_t filename_length = 0;
     const uint8_t* filetype_buffer = nullptr;
@@ -714,7 +718,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
     bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(),
         fp_length, session_data->file_octets[source_id], dir,
         get_header(source_id)->get_multi_file_processing_id(), file_position,
-        filename_buffer, filename_length, uri_buffer, uri_length, host);
+        filename_buffer, filename_length, uri_buffer, uri_length, host, is_partially_downloaded);
     if (continue_processing_file)
     {
         session_data->file_depth_remaining[source_id] -= fp_length;