From: Oleg Torubara -X (otorubar - SOFTSERVE INC at Cisco) Date: Mon, 10 Feb 2025 20:00:47 +0000 (+0000) Subject: Pull request #4604: file_api, http_inspect: add info about partial download to FileInfo X-Git-Tag: 3.7.1.0~30 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6883bf1a07a84df07773974ca17860481b076669;p=thirdparty%2Fsnort3.git Pull request #4604: file_api, http_inspect: add info about partial download to FileInfo Merge in SNORT/snort3 from ~OTORUBAR/snort3:partial_download to master Squashed commit of the following: commit db3bf9c938d28a59851e9a74969cb580a78fcf78 Author: otorubar Date: Fri Jan 31 10:20:46 2025 -0800 file_api, http_inspect: add info about partial download to FileInfo --- diff --git a/src/file_api/file_flows.cc b/src/file_api/file_flows.cc index eb3249645..3252d7b61 100644 --- a/src/file_api/file_flows.cc +++ b/src/file_api/file_flows.cc @@ -326,7 +326,7 @@ void FileFlows::remove_processed_file_context(uint64_t file_id) bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_data, int data_size, uint64_t offset, FileDirection dir, uint64_t multi_file_processing_id, FilePosition position, const uint8_t* fname, uint32_t name_size, - const uint8_t* url, uint32_t url_size, const std::string& host_name) + const uint8_t* url, uint32_t url_size, const std::string& host_name, const bool is_partial) { int64_t file_depth = FileService::get_max_file_depth(); bool continue_processing; @@ -351,9 +351,10 @@ bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_da return false; } - if (context->has_to_re_eval() and context->processing_complete) + if (context->has_to_re_eval() and context->processing_complete and not is_partial) context->reset(); + context->set_partial_flag(is_partial); context->set_weak_file_name((const char*)fname, name_size); context->set_weak_url((const char*)url, url_size); context->set_host(host_name.c_str(), host_name.size()); @@ -378,7 +379,7 @@ bool FileFlows::file_process(Packet* p, uint64_t file_id, const uint8_t* file_da context->set_file_id(file_id); } - if (context->is_cacheable() and not is_new_context) + if ((context->is_cacheable() and not is_new_context) or context->is_partial_download()) // If partial header was seen - check file_cache { FileVerdict verdict = FileService::get_file_cache()->cached_verdict_lookup(p, context, file_policy); diff --git a/src/file_api/file_flows.h b/src/file_api/file_flows.h index 1a0686ec5..c2f8efb37 100644 --- a/src/file_api/file_flows.h +++ b/src/file_api/file_flows.h @@ -97,7 +97,7 @@ public: bool file_process(Packet* p, uint64_t file_id, const uint8_t* file_data, int data_size, uint64_t offset, FileDirection, uint64_t multi_file_processing_id=0, FilePosition=SNORT_FILE_POSITION_UNKNOWN, const uint8_t* fname = nullptr, uint32_t name_size = 0, - const uint8_t* url = nullptr, uint32_t url_size = 0, const std::string& host_name = ""); + const uint8_t* url = nullptr, uint32_t url_size = 0, const std::string& host_name = "", const bool is_partial = false); static unsigned file_flow_data_id; diff --git a/src/file_api/file_lib.cc b/src/file_api/file_lib.cc index 09aa3a24d..5fb9dd2b3 100644 --- a/src/file_api/file_lib.cc +++ b/src/file_api/file_lib.cc @@ -137,6 +137,7 @@ void FileInfo::copy(const FileInfo& other, bool clear_data) host_set = other.host_set; verdict = other.verdict; file_type_enabled = other.file_type_enabled; + is_partial = other.is_partial; file_signature_enabled = other.file_signature_enabled; file_capture_enabled = other.file_capture_enabled; file_state = other.file_state; @@ -510,6 +511,16 @@ void FileContext::check_policy(Flow* flow, FileDirection dir, FilePolicyBase* po policy->policy_check(flow, this); } +void FileInfo::set_partial_flag(bool partial) +{ + is_partial = partial; +} + +bool FileInfo::is_partial_download() const +{ + return is_partial; +} + void FileInfo::reset() { verdict = FILE_VERDICT_UNKNOWN; @@ -1092,5 +1103,15 @@ TEST_CASE ("re_eval", "[file_info]") info.unset_re_eval(); CHECK (false == info.has_to_re_eval()); } + +TEST_CASE ("is_partial", "[file_info]") +{ + FI_TEST info; + CHECK (false == info.is_partial_download()); + info.set_partial_flag(true); + CHECK (true == info.is_partial_download()); + info.set_partial_flag(false); + CHECK (false == info.is_partial_download()); +} #endif diff --git a/src/file_api/file_lib.h b/src/file_api/file_lib.h index ef1d7f777..014e88873 100644 --- a/src/file_api/file_lib.h +++ b/src/file_api/file_lib.h @@ -111,9 +111,8 @@ public: bool has_to_re_eval(); void unset_re_eval(); - // Flag which indicates that the file requires re-eval even though it was fully processed before. - // If "true" and the policy was checked - has to be set to "false" again. - bool re_eval = false; + void set_partial_flag(bool partial); + bool is_partial_download() const; protected: std::string file_name; @@ -134,6 +133,12 @@ protected: FileState file_state = { FILE_CAPTURE_SUCCESS, FILE_SIG_PROCESSING }; uint32_t policy_id = 0; UserFileDataBase* user_file_data = nullptr; + + // Flag which indicates that the file requires re-eval even though it was fully processed before. + // If "true" and the policy was checked - has to be set to "false" again. + bool re_eval = false; + // Indicates that file transmission goes through 206 HTTP Partial Content + bool is_partial = false; }; class SO_PUBLIC FileContext : public FileInfo @@ -176,7 +181,6 @@ public: // Configuration functions void remove_segments(); void reset(); - private: uint64_t processed_bytes = 0; void* file_type_context; diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index 6ed1f03b8..cd13c5d74 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -603,6 +603,7 @@ static FilePosition find_range_file_pos(const std::string& hdr_content, bool fro if (errno or end_ptr != hdr_content.c_str() + dash_pos) return SNORT_FILE_POSITION_UNKNOWN; + // return middle no matter what range actually is if (range_start != 0) return SNORT_FILE_MIDDLE; @@ -652,6 +653,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data) const bool front = (body_octets == 0) && (session_data->partial_inspected_octets[source_id] == 0); const bool back = (session_data->cutter[source_id] == nullptr) || tcp_close; + bool is_partially_downloaded = false; if (session_data->status_code_num != 206 or source_id != SRC_SERVER) { @@ -680,6 +682,8 @@ void HttpMsgBody::do_file_processing(const Field& file_data) if (file_position == SNORT_FILE_POSITION_UNKNOWN) return; + + is_partially_downloaded = true; } const int32_t fp_length = (file_data.length() <= @@ -695,7 +699,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data) uint64_t file_index = get_header(source_id)->get_file_cache_index(); // Get host from the header field. std::string host = get_header(source_id)->get_host_header_field(); - + const uint8_t* filename_buffer = nullptr; uint32_t filename_length = 0; const uint8_t* filetype_buffer = nullptr; @@ -714,7 +718,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data) bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(), fp_length, session_data->file_octets[source_id], dir, get_header(source_id)->get_multi_file_processing_id(), file_position, - filename_buffer, filename_length, uri_buffer, uri_length, host); + filename_buffer, filename_length, uri_buffer, uri_length, host, is_partially_downloaded); if (continue_processing_file) { session_data->file_depth_remaining[source_id] -= fp_length;