From: Tom Peters (thopeter) Date: Thu, 28 Apr 2022 15:32:43 +0000 (+0000) Subject: Pull request #3396: Http mime depth X-Git-Tag: 3.1.29.0~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9278d27fe4d14e0b630233b6a914338cf514e688;p=thirdparty%2Fsnort3.git Pull request #3396: Http mime depth Merge in SNORT/snort3 from ~KATHARVE/snort3:http_mime_depth to master Squashed commit of the following: commit 0a8379db6e1fa866ddc327409324e2ef094f0fa1 Author: Katura Harvey Date: Wed Apr 20 12:21:33 2022 -0400 http_inspect: move mime processing outside of file and detect depth --- diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index 51c78e091..e2ca27238 100755 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -132,9 +132,17 @@ HttpInspect::HttpInspect(const HttpParaList* params_) : } } +HttpInspect::~HttpInspect() +{ + delete params->mime_decode_conf; + delete params; + delete script_finder; +} + bool HttpInspect::configure(SnortConfig* ) { params->js_norm_param.js_norm->configure(); + params->mime_decode_conf->sync_all_depths(); return true; } diff --git a/src/service_inspectors/http_inspect/http_inspect.h b/src/service_inspectors/http_inspect/http_inspect.h index 436a522a1..c6954b368 100644 --- a/src/service_inspectors/http_inspect/http_inspect.h +++ b/src/service_inspectors/http_inspect/http_inspect.h @@ -43,7 +43,7 @@ class HttpInspect : public snort::Inspector { public: HttpInspect(const HttpParaList* params_); - ~HttpInspect() override { delete params; delete script_finder; } + ~HttpInspect() override; bool get_buf(snort::InspectionBuffer::Type ibt, snort::Packet* p, snort::InspectionBuffer& b) override; diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc index 733322801..6c224dfe9 100755 --- a/src/service_inspectors/http_inspect/http_module.cc +++ b/src/service_inspectors/http_inspect/http_module.cc @@ -248,22 +248,18 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) else if (val.is("decompress_pdf")) { params->decompress_pdf = val.get_bool(); - params->mime_decode_conf.set_decompress_pdf(val.get_bool()); } else if (val.is("decompress_swf")) { params->decompress_swf = val.get_bool(); - params->mime_decode_conf.set_decompress_swf(val.get_bool()); } else if (val.is("decompress_zip")) { params->decompress_zip = val.get_bool(); - params->mime_decode_conf.set_decompress_zip(val.get_bool()); } else if (val.is("decompress_vba")) { params->decompress_vba = val.get_bool(); - params->mime_decode_conf.set_decompress_vba(val.get_bool()); } else if (val.is("script_detection")) { @@ -486,6 +482,13 @@ bool HttpModule::end(const char* fqn, int, SnortConfig*) params->script_detection_handle = script_detection_handle; prepare_http_header_list(params); + + params->mime_decode_conf = new DecodeConfig(); + params->mime_decode_conf->set_decompress_pdf(params->decompress_pdf); + params->mime_decode_conf->set_decompress_swf(params->decompress_swf); + params->mime_decode_conf->set_decompress_zip(params->decompress_zip); + params->mime_decode_conf->set_decompress_vba(params->decompress_vba); + return true; } diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h index ced003133..77063b916 100755 --- a/src/service_inspectors/http_inspect/http_module.h +++ b/src/service_inspectors/http_inspect/http_module.h @@ -58,7 +58,7 @@ public: bool decompress_swf = false; bool decompress_zip = false; bool decompress_vba = false; - snort::DecodeConfig mime_decode_conf; + snort::DecodeConfig* mime_decode_conf; bool script_detection = false; snort::LiteralSearch::Handle* script_detection_handle = nullptr; bool publish_request_body = true; diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index dd44d0460..151530257 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -142,8 +142,32 @@ void HttpMsgBody::analyze() msg_text_new.length() : pub_depth_remaining; pub_depth_remaining -= publish_length; } + + if (session_data->mime_state[source_id]) + { + // FIXIT-M this interface does not convey any indication of end of message body. If the + // message body ends in the middle of a MIME message the partial file will not be flushed. - if (session_data->file_depth_remaining[source_id] > 0 or + Packet* p = DetectionEngine::get_current_packet(); + const uint8_t* const section_end = msg_text_new.start() + msg_text_new.length(); + const uint8_t* ptr = msg_text_new.start(); + while (ptr < section_end) + { + // After process_mime_data(), ptr will point to the last byte processed in the current + // MIME part + ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr, + (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN); + ptr++; + } + + const BufferData& vba_buf = session_data->mime_state[source_id]->get_ole_buf(); + if (vba_buf.data_ptr()) + ole_data.set(vba_buf.length(), vba_buf.data_ptr()); + + detect_data.set(msg_text.length(), msg_text.start()); + } + + else if (session_data->file_depth_remaining[source_id] > 0 or session_data->detect_depth_remaining[source_id] > 0) { do_utf_decoding(msg_text_new, decoded_body); @@ -217,9 +241,8 @@ void HttpMsgBody::analyze() // file attachment body data. // FIXIT-E currently the file_data buffer is set to the body of the last attachment per // message section. - if (!session_data->mime_state[source_id]) - set_file_data(const_cast(detect_data.start()), - (unsigned)detect_data.length()); + set_file_data(const_cast(detect_data.start()), + (unsigned)detect_data.length()); } } body_octets += msg_text.length(); @@ -228,8 +251,7 @@ void HttpMsgBody::analyze() void HttpMsgBody::do_utf_decoding(const Field& input, Field& output) { - if ((session_data->mime_state[source_id] != nullptr) || - (session_data->utf_state[source_id] == nullptr) || (input.length() == 0)) + if ((session_data->utf_state[source_id] == nullptr) || (input.length() == 0)) { output.set(input); return; @@ -283,8 +305,7 @@ void HttpMsgBody::get_ole_data() void HttpMsgBody::do_file_decompression(const Field& input, Field& output) { - if ((session_data->mime_state[source_id] != nullptr) || - session_data->fd_state[source_id] == nullptr) + if (session_data->fd_state[source_id] == nullptr) { output.set(input); return; @@ -422,74 +443,49 @@ void HttpMsgBody::do_file_processing(const Field& file_data) return; } - if (!session_data->mime_state[source_id]) - { - const int32_t fp_length = (file_data.length() <= - session_data->file_depth_remaining[source_id]) ? - file_data.length() : session_data->file_depth_remaining[source_id]; + const int32_t fp_length = (file_data.length() <= + session_data->file_depth_remaining[source_id]) ? + file_data.length() : session_data->file_depth_remaining[source_id]; - FileFlows* file_flows = FileFlows::get_file_flows(flow); - if (!file_flows) - return; + FileFlows* file_flows = FileFlows::get_file_flows(flow); + if (!file_flows) + return; - const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD; + const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD; - const uint64_t file_index = get_header(source_id)->get_file_cache_index(); + const uint64_t file_index = get_header(source_id)->get_file_cache_index(); - bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(), - fp_length, session_data->file_octets[source_id], dir, - get_header(source_id)->get_multi_file_processing_id(), file_position); - if (continue_processing_file) - { - session_data->file_depth_remaining[source_id] -= fp_length; + bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(), + fp_length, session_data->file_octets[source_id], dir, + get_header(source_id)->get_multi_file_processing_id(), file_position); + if (continue_processing_file) + { + session_data->file_depth_remaining[source_id] -= fp_length; - // With the first piece of the file we must provide the filename and URI - if (front) + // With the first piece of the file we must provide the filename and URI + if (front) + { + if (request != nullptr) { - if (request != nullptr) - { - const uint8_t* filename_buffer; - const uint8_t* uri_buffer; - uint32_t filename_length; - uint32_t uri_length; - get_file_info(dir, filename_buffer, filename_length, uri_buffer, uri_length); - - continue_processing_file = file_flows->set_file_name(filename_buffer, - filename_length, 0, - get_header(source_id)->get_multi_file_processing_id(), uri_buffer, - uri_length); - } + const uint8_t* filename_buffer; + const uint8_t* uri_buffer; + uint32_t filename_length; + uint32_t uri_length; + get_file_info(dir, filename_buffer, filename_length, uri_buffer, uri_length); + + continue_processing_file = file_flows->set_file_name(filename_buffer, + filename_length, 0, + get_header(source_id)->get_multi_file_processing_id(), uri_buffer, + uri_length); } } - if (!continue_processing_file) - { - // file processing doesn't want any more data - session_data->file_depth_remaining[source_id] = 0; - } - session_data->file_octets[source_id] += fp_length; } - else + if (!continue_processing_file) { - // FIXIT-M this interface does not convey any indication of end of message body. If the - // message body ends in the middle of a MIME message the partial file will not be flushed. - - const uint8_t* const section_end = file_data.start() + file_data.length(); - const uint8_t* ptr = file_data.start(); - while (ptr < section_end) - { - // After process_mime_data(), ptr will point to the last byte processed in the current - // MIME part - ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr, - (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN); - ptr++; - } - - const BufferData& vba_buf = session_data->mime_state[source_id]->get_ole_buf(); - if (vba_buf.data_ptr()) - ole_data.set(vba_buf.length(), vba_buf.data_ptr()); - - session_data->file_octets[source_id] += file_data.length(); + // file processing doesn't want any more data + session_data->file_depth_remaining[source_id] = 0; } + session_data->file_octets[source_id] += fp_length; } // Parses out the filename and URI associated with this file. diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index c0c7687b1..3e9de4699 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -443,15 +443,26 @@ void HttpMsgHeader::update_flow() } } -// Common activities of preparing for upcoming regular body or chunked body +// Common activities of preparing for upcoming body void HttpMsgHeader::prepare_body() { session_data->body_octets[source_id] = 0; - const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth : - params->response_depth; - session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX; - params->js_norm_param.js_norm->set_detection_depth(session_data->detect_depth_remaining[source_id]); - + setup_mime(); + if (!session_data->mime_state[source_id]) + { + const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth : + params->response_depth; + session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX; + params->js_norm_param.js_norm->set_detection_depth(session_data->detect_depth_remaining[source_id]); + } + else + { + // File and decode depths are per attachment, so if either is greater than 0 we inspect the + // full message body. Currently the decode depths are not configurable for http_inspect so + // are always the default of unlimited, meaning for MIME we always inspect the full message + // body + session_data->detect_depth_remaining[source_id] = INT64_MAX; + } if ((source_id == SRC_CLIENT) and params->publish_request_body and session_data->for_http2) { session_data->publish_octets[source_id] = 0; @@ -479,20 +490,8 @@ void HttpMsgHeader::prepare_body() } } -void HttpMsgHeader::setup_file_processing() +void HttpMsgHeader::setup_mime() { - session_data->file_octets[source_id] = 0; - - const int64_t max_file_depth = FileService::get_max_file_depth(); - if (max_file_depth <= 0) - { - session_data->file_depth_remaining[source_id] = 0; - return; - } - - // Generate the unique file id for multi file processing - set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id()); - // Do we meet all the conditions for MIME file processing? if (source_id == SRC_CLIENT) { @@ -501,15 +500,18 @@ void HttpMsgHeader::setup_file_processing() { if (boundary_present(content_type)) { + // Generate the unique file id for multi file processing + set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id()); + Packet* p = DetectionEngine::get_current_packet(); const Field& uri = request->get_uri_norm_classic(); if (uri.length() > 0) session_data->mime_state[source_id] = new MimeSession(p, - ¶ms->mime_decode_conf, &mime_conf, get_multi_file_processing_id(), + params->mime_decode_conf, &mime_conf, get_multi_file_processing_id(), uri.start(), uri.length()); else session_data->mime_state[source_id] = new MimeSession(p, - ¶ms->mime_decode_conf, &mime_conf, get_multi_file_processing_id()); + params->mime_decode_conf, &mime_conf, get_multi_file_processing_id()); // Show file processing the Content-Type header as if it were regular data. // This will enable it to find the boundary string. @@ -525,15 +527,28 @@ void HttpMsgHeader::setup_file_processing() } } } +} - // Otherwise do regular file processing - if (session_data->mime_state[source_id] == nullptr) +void HttpMsgHeader::setup_file_processing() +{ + if (session_data->mime_state[source_id]) + return; + + session_data->file_octets[source_id] = 0; + const int64_t max_file_depth = FileService::get_max_file_depth(); + if (max_file_depth <= 0) { - session_data->file_depth_remaining[source_id] = max_file_depth; - FileFlows* file_flows = FileFlows::get_file_flows(flow); - if (!file_flows) - session_data->file_depth_remaining[source_id] = 0; + session_data->file_depth_remaining[source_id] = 0; + return; } + + // Generate the unique file id for multi file processing + set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id()); + + session_data->file_depth_remaining[source_id] = max_file_depth; + FileFlows* file_flows = FileFlows::get_file_flows(flow); + if (!file_flows) + session_data->file_depth_remaining[source_id] = 0; } void HttpMsgHeader::setup_encoding_decompression() diff --git a/src/service_inspectors/http_inspect/http_msg_header.h b/src/service_inspectors/http_inspect/http_msg_header.h index 64df1329e..06f9fc3d3 100644 --- a/src/service_inspectors/http_inspect/http_msg_header.h +++ b/src/service_inspectors/http_inspect/http_msg_header.h @@ -53,6 +53,7 @@ public: private: void prepare_body(); + void setup_mime(); void setup_file_processing(); void setup_encoding_decompression(); void setup_utf_decoding();