From 941a638ffcc024f463bbf32e08c46d5dc31bdccb Mon Sep 17 00:00:00 2001 From: "Mike Stepanek (mstepane)" Date: Thu, 16 Jan 2020 16:11:34 +0000 Subject: [PATCH] Merge pull request #1931 in SNORT/snort3 from ~KATHARVE/snort3:http_mime_file to master Squashed commit of the following: commit 159de978a1e07db74de106810e3f268690488567 Author: Katura Harvey Date: Wed Jan 8 15:50:36 2020 -0500 mime: support simultaneous file processing of MIME-encoded files over HTTP/1.1 --- src/mime/file_mime_process.cc | 80 ++++++++++++++++--- src/mime/file_mime_process.h | 12 ++- .../http_inspect/http_msg_header.cc | 3 +- .../http_stream_splitter_finish.cc | 12 ++- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc index 2281877a0..e25a82889 100644 --- a/src/mime/file_mime_process.cc +++ b/src/mime/file_mime_process.cc @@ -28,6 +28,7 @@ #include "detection/detection_engine.h" #include "file_api/file_flows.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "search_engines/search_tool.h" #include "utils/util_cstring.h" @@ -597,13 +598,8 @@ const uint8_t* MimeSession::process_mime_data_paf( } /*Process file type/file signature*/ - FileFlows* file_flows = FileFlows::get_file_flows(flow); - if (file_flows && file_flows->file_process(p, buffer, buf_size, position, upload) - && (isFileStart(position)) && log_state) - { - file_flows->set_file_name((const uint8_t*)filename.c_str(), filename.length()); - filename.clear(); - } + mime_file_process(p, buffer, buf_size, position, upload); + if (mime_stats) { switch (decode_state->get_decode_type()) @@ -639,12 +635,20 @@ const uint8_t* MimeSession::process_mime_data_paf( return end; } +void MimeSession::reset_file_data() +{ + // Clear MIME's file data to prepare for next file + file_counter++; + file_process_offset = 0; + current_mime_file_id = 0; + continue_inspecting_file = true; +} + // Main function for mime processing // This should be called when mime data is available const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, int data_size, bool upload, FilePosition position) { - Flow* flow = p->flow; const uint8_t* attach_start = start; const uint8_t* attach_end; @@ -652,12 +656,14 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, if (position != SNORT_FILE_POSITION_UNKNOWN) { + if (position == SNORT_FILE_START or position == SNORT_FILE_FULL) + reset_file_data(); process_mime_data_paf(p, attach_start, data_end_marker, upload, position); return data_end_marker; } - initFilePosition(&position, get_file_processed_size(flow)); + initFilePosition(&position, file_process_offset); /* look for boundary */ while (start < data_end_marker) { @@ -668,6 +674,7 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, finalFilePosition(&position); process_mime_data_paf(p, attach_start, attach_end, upload, position); + reset_file_data(); data_state = STATE_MIME_HEADER; position = SNORT_FILE_START; attach_start = start + 1; @@ -678,7 +685,7 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, if ((start == data_end_marker) && (attach_start < data_end_marker)) { - updateFilePosition(&position, get_file_processed_size(flow)); + updateFilePosition(&position, file_process_offset); process_mime_data_paf(p, attach_start, data_end_marker, upload, position); } @@ -800,11 +807,12 @@ void MimeSession::exit() delete mime_hdr_search_mpse; } -MimeSession::MimeSession(DecodeConfig* dconf, MailLogConfig* lconf) +MimeSession::MimeSession(DecodeConfig* dconf, MailLogConfig* lconf, uint64_t base_file_id) { decode_conf = dconf; log_config = lconf; log_state = new MailLogState(log_config); + session_base_file_id = base_file_id; reset_mime_paf_state(&mime_boundary); } @@ -817,3 +825,53 @@ MimeSession::~MimeSession() delete(log_state); } +uint64_t MimeSession::get_mime_file_id() +{ + if (!current_mime_file_id) + { + const int data_len = sizeof(session_base_file_id) + sizeof(file_counter); + uint8_t data[data_len]; + memcpy(data, (void*)&session_base_file_id, sizeof(session_base_file_id)); + memcpy(data + sizeof(session_base_file_id), (void*)&file_counter, sizeof(file_counter)); + current_mime_file_id = str_to_hash(data, data_len); + } + return current_mime_file_id; +} + +void MimeSession::mime_file_process(Packet* p, const uint8_t* data, int data_size, + FilePosition position, bool upload) +{ + Flow* flow = p->flow; + FileFlows* file_flows = FileFlows::get_file_flows(flow); + if(!file_flows) + return; + + if (continue_inspecting_file or position == SNORT_FILE_END) + { + if (session_base_file_id) + { + const FileDirection dir = upload? FILE_UPLOAD : FILE_DOWNLOAD; + uint64_t offset = file_process_offset; + // MIME has found the end of a file that file processing didn't want - tell file + // processing it can clear this file's data + if (!continue_inspecting_file) + offset = 0; + uint64_t file_id = get_mime_file_id(); + continue_inspecting_file = file_flows->file_process(p, file_id, data, data_size, offset, + dir, file_id, position); + } + else + { + continue_inspecting_file = file_flows->file_process(p, data, data_size, position, + upload); + } + file_process_offset += data_size; + if (continue_inspecting_file and (isFileStart(position)) && log_state) + { + file_flows->set_file_name((const uint8_t*)filename.c_str(), filename.length()); + filename.clear(); + } + } + if (position == SNORT_FILE_FULL or position == SNORT_FILE_END) + reset_file_data(); +} diff --git a/src/mime/file_mime_process.h b/src/mime/file_mime_process.h index 549c8151c..682c2ee7d 100644 --- a/src/mime/file_mime_process.h +++ b/src/mime/file_mime_process.h @@ -55,7 +55,7 @@ namespace snort class SO_PUBLIC MimeSession { public: - MimeSession(DecodeConfig*, MailLogConfig*); + MimeSession(DecodeConfig*, MailLogConfig*, uint64_t base_file_id=0); virtual ~MimeSession(); MimeSession(const MimeSession&) = delete; @@ -84,6 +84,16 @@ private: MailLogState* log_state = nullptr; MimeStats* mime_stats = nullptr; std::string filename; + bool continue_inspecting_file = true; + // This counter is not an accurate count of files; used only for creating a unique mime_file_id + uint32_t file_counter = 0; + uint32_t file_process_offset = 0; + uint64_t session_base_file_id = 0; + uint64_t current_mime_file_id = 0; + uint64_t get_mime_file_id(); + void mime_file_process(Packet* p, const uint8_t* data, int data_size, + FilePosition position, bool upload); + void reset_file_data(); // SMTP, IMAP, POP might have different implementation for this virtual int handle_header_line(const uint8_t*, const uint8_t*, int, Packet*) { return 0; } diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index 1bcd3bf3e..6cdeb594d 100644 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -329,7 +329,8 @@ void HttpMsgHeader::setup_file_processing() { if (boundary_present(content_type)) { - session_data->mime_state[source_id] = new MimeSession(&decode_conf, &mime_conf); + session_data->mime_state[source_id] = new MimeSession(&decode_conf, &mime_conf, + transaction->get_file_processing_id(source_id)); // Show file processing the Content-Type header as if it were regular data. // This will enable it to find the boundary string. // FIXIT-L develop a proper interface for passing the boundary string. diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index 007ecccc4..698b2baa8 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -102,6 +102,8 @@ bool HttpStreamSplitter::finish(Flow* flow) return true; } + // FIXIT-M No longer necessary to send an empty body section because the header section is + // always forwarded to detection. // If the message has been truncated immediately following the start line or immediately // following the headers (a body was expected) then we need to process an empty section to // provide an inspection section. Otherwise the start line and headers won't go through @@ -130,10 +132,12 @@ bool HttpStreamSplitter::finish(Flow* flow) if (!session_data->mime_state[source_id]) { FileFlows* file_flows = FileFlows::get_file_flows(flow); - const bool download = (source_id == SRC_SERVER); + const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD; size_t file_index = 0; + uint64_t file_processing_id = 0; + // FIXIT-L How can there be a file in progress and no transaction in this direction? if (session_data->transaction[source_id] != nullptr) { HttpMsgRequest* request = session_data->transaction[source_id]->get_request(); @@ -141,9 +145,11 @@ bool HttpStreamSplitter::finish(Flow* flow) { file_index = request->get_http_uri()->get_file_proc_hash(); } + file_processing_id = + session_data->transaction[source_id]->get_file_processing_id(source_id); } - - file_flows->file_process(packet, nullptr, 0, SNORT_FILE_END, !download, file_index); + file_flows->file_process(packet, file_index, nullptr, 0, 0, dir, file_processing_id, + SNORT_FILE_END); } else { -- 2.47.3