From: Tom Peters (thopeter) Date: Tue, 8 Mar 2022 04:19:54 +0000 (+0000) Subject: Pull request #3281: http_inspect: call mime in a loop for each attachment X-Git-Tag: 3.1.25.0~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=db3c28d12d1bda475399fec3c3f9a1bb6bff869f;p=thirdparty%2Fsnort3.git Pull request #3281: http_inspect: call mime in a loop for each attachment Merge in SNORT/snort3 from ~KATHARVE/snort3:http_mime_file_data_part1 to master Squashed commit of the following: commit f9a0cd0d24bb4730037aa8d426859556f09a8ab8 Author: Katura Harvey Date: Thu Mar 3 13:37:58 2022 -0500 http_inspect: use http_inspect decompression config parameters for HTTP MIME traffic instead of file_id; file_id: remove unused decompression and decode depth parameters commit c77e3b165142f89a78d4c60cce25962f00f13a1d Author: Katura Harvey Date: Thu Feb 17 17:47:04 2022 -0500 mime: fix resetting state after every attachment and check state instead of decode object commit 70a27c3a2cc5866a5ca38e5350b3575543b68d4e Author: Katura Harvey Date: Wed Feb 16 17:16:38 2022 -0500 http_inspect: call mime in a loop for each attachment mime: return at the end of each attachment and set the file_data for http --- diff --git a/doc/user/http_inspect.txt b/doc/user/http_inspect.txt index cab6882d7..98dec3b94 100755 --- a/doc/user/http_inspect.txt +++ b/doc/user/http_inspect.txt @@ -154,27 +154,32 @@ feature is on by default: normalize_utf = false will deactivate it. ===== decompress_pdf decompress_pdf = true will enable decompression of compressed portions of -PDF files encountered in a response body. http_inspect will examine the -response body for PDF files that are then parsed to locate PDF streams with +PDF files encountered in a message body. http_inspect will examine the +message body for PDF files that are then parsed to locate PDF streams with a single /FlateDecode filter. The compressed content is decompressed and made available through the file data rule option. ===== decompress_swf decompress_swf = true will enable decompression of compressed SWF (Adobe -Flash content) files encountered in a response body. The available +Flash content) files encountered in a message body. The available decompression modes are ’deflate’ and ’lzma’. http_inspect will search for the file signatures CWS for Deflate/ZLIB and ZWS for LZMA. The compressed content is decompressed and made available through the file data rule option. The compressed SWF file signature is converted to FWS to indicate an uncompressed file. +===== decompress_zip + +decompress_zip = true will enable decompression of compressed zip archives encountered in a message +body. The compressed content is decompressed and made available through the file_data rule option. + ===== decompress_vba decompress_vba = true will enable decompression of RLE (Run Length Encoding) compressed vba (Visual Basic for Applications) macro data of MS Office -files. The MS office files are PKZIP compressed which are parsed to locate -the OLE (Object Linking and Embedding) file embedded with the files +files encountered in a message body. The MS office files are PKZIP compressed which are parsed to +locate the OLE (Object Linking and Embedding) file embedded with the files containing RLE compressed vba macro data. The decompressed vba macro data is then made available through the vba_data ips rule option. diff --git a/doc/user/pop_imap.txt b/doc/user/pop_imap.txt index 6d5c76cac..b49363118 100644 --- a/doc/user/pop_imap.txt +++ b/doc/user/pop_imap.txt @@ -15,7 +15,7 @@ POP inspector and IMAP inspector offer same set of configuration options for MIME decoding depth. These depths range from 0 to 65535 bytes. Setting the value to 0 ("do none") turns the feature off. Alternatively the value -1 means an unlimited amount of data should be decoded. If you do not -specify the default value is 1460 bytes. +specify the default value is -1 (unlimited). The depth limits apply per attachment. They are: diff --git a/src/file_api/file_module.cc b/src/file_api/file_module.cc index 9f86a05f3..c38214368 100644 --- a/src/file_api/file_module.cc +++ b/src/file_api/file_module.cc @@ -132,30 +132,9 @@ static const Parameter file_id_params[] = { "trace_stream", Parameter::PT_BOOL, nullptr, "false", "enable runtime dump of file data" }, - { "b64_decode_depth", Parameter::PT_INT, "-1:65535", "-1", - "base64 decoding depth (-1 no limit)" }, - - { "bitenc_decode_depth", Parameter::PT_INT, "-1:65535", "-1", - "Non-Encoded MIME attachment extraction depth (-1 no limit)" }, - - { "decompress_pdf", Parameter::PT_BOOL, nullptr, "false", - "decompress pdf files" }, - - { "decompress_swf", Parameter::PT_BOOL, nullptr, "false", - "decompress swf files" }, - - { "decompress_zip", Parameter::PT_BOOL, nullptr, "false", - "decompress zip files" }, - { "decompress_buffer_size", Parameter::PT_INT, "1024:max31", "100000", "file decompression buffer size" }, - { "qp_decode_depth", Parameter::PT_INT, "-1:65535", "-1", - "Quoted Printable decoding depth (-1 no limit)" }, - - { "uu_decode_depth", Parameter::PT_INT, "-1:65535", "-1", - "Unix-to-Unix decoding depth (-1 no limit)" }, - { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -259,43 +238,9 @@ bool FileIdModule::set(const char*, Value& v, SnortConfig*) else if ( v.is("trace_stream") ) fc->trace_stream = v.get_bool(); - else if ( v.is("decompress_pdf") ) - FileService::decode_conf.set_decompress_pdf(v.get_bool()); - - else if ( v.is("decompress_swf") ) - FileService::decode_conf.set_decompress_swf(v.get_bool()); - - else if ( v.is("decompress_zip") ) - FileService::decode_conf.set_decompress_zip(v.get_bool()); - else if ( v.is("decompress_buffer_size") ) FileService::decode_conf.set_decompress_buffer_size(v.get_uint32()); - else if (v.is("b64_decode_depth")) - { - int32_t value = v.get_int32(); - int32_t mime = value > 0 ? value : -(value+1); - FileService::decode_conf.set_b64_depth(mime); - } - else if (v.is("bitenc_decode_depth")) - { - int32_t value = v.get_int32(); - int32_t mime = value > 0 ? value : -(value+1); - FileService::decode_conf.set_bitenc_depth(mime); - } - else if (v.is("qp_decode_depth")) - { - int32_t value = v.get_int32(); - int32_t mime = value > 0 ? value : -(value+1); - FileService::decode_conf.set_qp_depth(mime); - } - else if (v.is("uu_decode_depth")) - { - int32_t value = v.get_int32(); - int32_t mime = value > 0 ? value : -(value+1); - FileService::decode_conf.set_uu_depth(mime); - } - else if ( v.is("rev") ) rule.rev = v.get_uint32(); diff --git a/src/mime/file_mime_config.h b/src/mime/file_mime_config.h index d0b3f9408..0a88c6d54 100644 --- a/src/mime/file_mime_config.h +++ b/src/mime/file_mime_config.h @@ -26,7 +26,7 @@ /*These are temporary values*/ #define DEFAULT_MIME_MEMCAP 838860 -#define DEFAULT_DEPTH 1464 +#define DEFAULT_DEPTH 0 #define DEFAULT_DECOMP 100000 #define MAX_LOG_MEMCAP 104857600 #define MIN_LOG_MEMCAP 3276 diff --git a/src/mime/file_mime_decode.cc b/src/mime/file_mime_decode.cc index 6e301f33b..43b4a5cfc 100644 --- a/src/mime/file_mime_decode.cc +++ b/src/mime/file_mime_decode.cc @@ -270,7 +270,7 @@ void MimeDecode::file_decomp_init() (void)File_Decomp_Init(fd_state); } -MimeDecode::MimeDecode(DecodeConfig* conf) +MimeDecode::MimeDecode(const DecodeConfig* conf) { config = conf; file_decomp_init(); diff --git a/src/mime/file_mime_decode.h b/src/mime/file_mime_decode.h index e19d109e0..3851c693c 100644 --- a/src/mime/file_mime_decode.h +++ b/src/mime/file_mime_decode.h @@ -57,7 +57,7 @@ struct MimeStats class SO_PUBLIC MimeDecode { public: - MimeDecode(snort::DecodeConfig* conf); + MimeDecode(const snort::DecodeConfig* conf); ~MimeDecode(); // get the decode type from buffer @@ -90,7 +90,7 @@ public: private: void get_ole_data(); DecodeType decode_type = DECODE_NONE; - snort::DecodeConfig* config; + const snort::DecodeConfig* config; DataDecode* decoder = nullptr; fd_session_t* fd_state = nullptr; BufferData ole_data; diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc index c31ab9098..d5cdd86a5 100644 --- a/src/mime/file_mime_process.cc +++ b/src/mime/file_mime_process.cc @@ -154,7 +154,7 @@ void MimeSession::setup_decode(const char* data, int size, bool cnt_xf) if (decode_state != nullptr) { decode_state->process_decode_type(data, size, cnt_xf, mime_stats); - state_flags |= MIME_FLAG_EMAIL_ATTACH; + state_flags |= MIME_FLAG_FILE_ATTACH; } } } @@ -350,7 +350,7 @@ const uint8_t* MimeSession::process_mime_header(Packet* p, const uint8_t* ptr, (MIME_FLAG_IN_CONTENT_TYPE | MIME_FLAG_FOLDING)) == MIME_FLAG_IN_CONTENT_TYPE) { if ((data_state == STATE_MIME_HEADER) && !(state_flags & - MIME_FLAG_EMAIL_ATTACH)) + MIME_FLAG_FILE_ATTACH)) { setup_decode((const char*)content_type_ptr, (eolm - content_type_ptr), false); } @@ -448,14 +448,14 @@ static const uint8_t* GetDataEnd(const uint8_t* data_start, * @return i index into p->payload where we stopped looking at data */ const uint8_t* MimeSession::process_mime_body(const uint8_t* ptr, - const uint8_t* data_end, bool is_data_end) + const uint8_t* data_end, bool is_body_end) { - if (state_flags & MIME_FLAG_EMAIL_ATTACH) + if (state_flags & MIME_FLAG_FILE_ATTACH) { const uint8_t* attach_start = ptr; const uint8_t* attach_end; - if (is_data_end ) + if (is_body_end ) { attach_end = GetDataEnd(ptr, data_end); } @@ -473,10 +473,9 @@ const uint8_t* MimeSession::process_mime_body(const uint8_t* ptr, } } - if (is_data_end) + if (is_body_end) { data_state = STATE_MIME_HEADER; - state_flags &= ~MIME_FLAG_EMAIL_ATTACH; } return data_end; @@ -562,65 +561,65 @@ const uint8_t* MimeSession::process_mime_data_paf( break; case STATE_DATA_BODY: start = process_mime_body(start, end, isFileEnd(position) ); - break; - } - } - /* We have either reached the end of MIME header or end of MIME encoded data*/ + if (state_flags & MIME_FLAG_FILE_ATTACH) + { + const DecodeConfig* conf = decode_conf; + const uint8_t* buffer = nullptr; + uint32_t buf_size = 0; - if ((decode_state) != nullptr) - { - DecodeConfig* conf = decode_conf; - const uint8_t* buffer = nullptr; - uint32_t buf_size = 0; + decode_state->get_decoded_data(&buffer, &buf_size); - decode_state->get_decoded_data(&buffer, &buf_size); + if (conf and buf_size > 0) + { + const uint8_t* decomp_buffer = nullptr; + uint32_t detection_size, decomp_buf_size = 0; - if (conf) - { - const uint8_t* decomp_buffer = nullptr; - uint32_t detection_size, decomp_buf_size = 0; + detection_size = (uint32_t)decode_state->get_detection_depth(); - detection_size = (uint32_t)decode_state->get_detection_depth(); + DecodeResult result = decode_state->decompress_data( + buffer, detection_size, decomp_buffer, decomp_buf_size + ); - DecodeResult result = decode_state->decompress_data( - buffer, detection_size, decomp_buffer, decomp_buf_size - ); + if ( result != DECODE_SUCCESS ) + decompress_alert(); - if ( result != DECODE_SUCCESS ) - decompress_alert(); + set_file_data(decomp_buffer, decomp_buf_size); + } - if (!is_http) - set_file_data(decomp_buffer, decomp_buf_size); - } + /*Process file type/file signature*/ + mime_file_process(p, buffer, buf_size, position, upload); - /*Process file type/file signature*/ - mime_file_process(p, buffer, buf_size, position, upload); + if (mime_stats) + { + switch (decode_state->get_decode_type()) + { + case DECODE_B64: + mime_stats->b64_bytes += buf_size; + break; + case DECODE_QP: + mime_stats->qp_bytes += buf_size; + break; + case DECODE_UU: + mime_stats->uu_bytes += buf_size; + break; + case DECODE_BITENC: + mime_stats->bitenc_bytes += buf_size; + break; + default: + break; + } + } - if (mime_stats) - { - switch (decode_state->get_decode_type()) - { - case DECODE_B64: - mime_stats->b64_bytes += buf_size; - break; - case DECODE_QP: - mime_stats->qp_bytes += buf_size; - break; - case DECODE_UU: - mime_stats->uu_bytes += buf_size; - break; - case DECODE_BITENC: - mime_stats->bitenc_bytes += buf_size; - break; - default: - break; + decode_state->reset_decoded_bytes(); } + break; } - - decode_state->reset_decoded_bytes(); } + if (isFileEnd(position)) + reset_part_state(); + /* if we got the data end reset state, otherwise we're probably still in the data * * to expect more data in next packet */ if (done_data) @@ -632,8 +631,12 @@ const uint8_t* MimeSession::process_mime_data_paf( return end; } -void MimeSession::reset_file_data() +void MimeSession::reset_part_state() { + state_flags = 0; + if (decode_state) + decode_state->clear_decode_state(); + // Clear MIME's file data to prepare for next file file_counter++; file_process_offset = 0; @@ -654,8 +657,6 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, if (position != SNORT_FILE_POSITION_UNKNOWN) { - if (position == SNORT_FILE_START or position == SNORT_FILE_FULL) - reset_file_data(); process_mime_data_paf(p, attach_start, data_end_marker, upload, position); return data_end_marker; @@ -672,10 +673,9 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start, finalFilePosition(&position); process_mime_data_paf(p, attach_start, attach_end, upload, position); - reset_file_data(); data_state = STATE_MIME_HEADER; position = SNORT_FILE_START; - attach_start = start + 1; + return attach_end; } start++; @@ -806,12 +806,11 @@ void MimeSession::exit() delete mime_hdr_search_mpse; } -MimeSession::MimeSession(Packet* p, DecodeConfig* dconf, MailLogConfig* lconf, uint64_t base_file_id, - bool session_is_http, const uint8_t* uri, const int32_t uri_length): +MimeSession::MimeSession(Packet* p, const DecodeConfig* dconf, MailLogConfig* lconf, uint64_t base_file_id, + const uint8_t* uri, const int32_t uri_length): decode_conf(dconf), log_config(lconf), log_state(new MailLogState(log_config)), - is_http(session_is_http), session_base_file_id(base_file_id), uri(uri), uri_length(uri_length) @@ -881,6 +880,4 @@ void MimeSession::mime_file_process(Packet* p, const uint8_t* data, int data_siz filename.clear(); } } - if (position == SNORT_FILE_FULL or position == SNORT_FILE_END) - reset_file_data(); } diff --git a/src/mime/file_mime_process.h b/src/mime/file_mime_process.h index 67599df8d..94ddd2ce1 100644 --- a/src/mime/file_mime_process.h +++ b/src/mime/file_mime_process.h @@ -38,7 +38,7 @@ namespace snort #define MIME_FLAG_GOT_BOUNDARY 0x00000004 #define MIME_FLAG_DATA_HEADER_CONT 0x00000008 #define MIME_FLAG_IN_CONT_TRANS_ENC 0x00000010 -#define MIME_FLAG_EMAIL_ATTACH 0x00000020 +#define MIME_FLAG_FILE_ATTACH 0x00000020 #define MIME_FLAG_MULTIPLE_EMAIL_ATTACH 0x00000040 #define MIME_FLAG_MIME_END 0x00000080 #define MIME_FLAG_IN_CONT_DISP 0x00000200 @@ -55,8 +55,8 @@ namespace snort class SO_PUBLIC MimeSession { public: - MimeSession(Packet*, DecodeConfig*, MailLogConfig*, uint64_t base_file_id=0, - bool session_is_http=false, const uint8_t* uri=nullptr, const int32_t uri_length=0); + MimeSession(Packet*, const DecodeConfig*, MailLogConfig*, uint64_t base_file_id=0, + const uint8_t* uri=nullptr, const int32_t uri_length=0); virtual ~MimeSession(); MimeSession(const MimeSession&) = delete; @@ -82,12 +82,11 @@ private: int data_state = STATE_DATA_INIT; int state_flags = 0; MimeDataPafInfo mime_boundary; - DecodeConfig* decode_conf = nullptr; + const DecodeConfig* decode_conf = nullptr; MailLogConfig* log_config = nullptr; MailLogState* log_state = nullptr; MimeStats* mime_stats = nullptr; std::string filename; - bool is_http; bool continue_inspecting_file = true; // This counter is not an accurate count of files; used only for creating a unique mime_file_id uint32_t file_counter = 0; @@ -101,7 +100,7 @@ private: uint64_t get_multiprocessing_file_id(); void mime_file_process(Packet* p, const uint8_t* data, int data_size, FilePosition position, bool upload); - void reset_file_data(); + void reset_part_state(); // SMTP, IMAP, POP might have different implementation for this virtual int handle_header_line(const uint8_t*, const uint8_t*, int, Packet*) { return 0; } diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc index 5c4dfb395..733322801 100755 --- a/src/service_inspectors/http_inspect/http_module.cc +++ b/src/service_inspectors/http_inspect/http_module.cc @@ -248,18 +248,22 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) else if (val.is("decompress_pdf")) { params->decompress_pdf = val.get_bool(); + params->mime_decode_conf.set_decompress_pdf(val.get_bool()); } else if (val.is("decompress_swf")) { params->decompress_swf = val.get_bool(); + params->mime_decode_conf.set_decompress_swf(val.get_bool()); } else if (val.is("decompress_zip")) { params->decompress_zip = val.get_bool(); + params->mime_decode_conf.set_decompress_zip(val.get_bool()); } else if (val.is("decompress_vba")) { params->decompress_vba = val.get_bool(); + params->mime_decode_conf.set_decompress_vba(val.get_bool()); } else if (val.is("script_detection")) { diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h index 15455f2b8..ced003133 100755 --- a/src/service_inspectors/http_inspect/http_module.h +++ b/src/service_inspectors/http_inspect/http_module.h @@ -26,6 +26,7 @@ #include "framework/module.h" #include "helpers/literal_search.h" +#include "mime/file_mime_config.h" #include "profiler/profiler.h" #include "http_enum.h" @@ -57,6 +58,7 @@ public: bool decompress_swf = false; bool decompress_zip = false; bool decompress_vba = false; + snort::DecodeConfig mime_decode_conf; bool script_detection = false; snort::LiteralSearch::Handle* script_detection_handle = nullptr; bool publish_request_body = true; diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index b4f306c44..ed5bc5248 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -212,8 +212,13 @@ void HttpMsgBody::analyze() partial_js_detect_length = js_norm_body.length(); } - set_file_data(const_cast(detect_data.start()), - (unsigned)detect_data.length()); + // If this is a MIME upload, the MIME library sets the file_data buffer to the + // file attachment body data. + // FIXIT-E currently the file_data buffer is set to the body of the last attachment per + // message section. + if (!session_data->mime_state[source_id]) + set_file_data(const_cast(detect_data.start()), + (unsigned)detect_data.length()); } } body_octets += msg_text.length(); @@ -463,8 +468,18 @@ void HttpMsgBody::do_file_processing(const Field& file_data) { // FIXIT-M this interface does not convey any indication of end of message body. If the // message body ends in the middle of a MIME message the partial file will not be flushed. - session_data->mime_state[source_id]->process_mime_data(p, file_data.start(), - file_data.length(), true, SNORT_FILE_POSITION_UNKNOWN); + + const uint8_t* const section_end = file_data.start() + file_data.length(); + const uint8_t* ptr = file_data.start(); + while (ptr < section_end) + { + // After process_mime_data(), ptr will point to the last byte processed in the current + // MIME part + ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr, + (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN); + ptr++; + } + session_data->file_octets[source_id] += file_data.length(); } } diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index 6a9a3fc35..115168a0d 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -505,12 +505,11 @@ void HttpMsgHeader::setup_file_processing() const Field& uri = request->get_uri_norm_classic(); if (uri.length() > 0) session_data->mime_state[source_id] = new MimeSession(p, - &FileService::decode_conf, &mime_conf, get_multi_file_processing_id(), - true, uri.start(), uri.length()); + ¶ms->mime_decode_conf, &mime_conf, get_multi_file_processing_id(), + uri.start(), uri.length()); else session_data->mime_state[source_id] = new MimeSession(p, - &FileService::decode_conf, &mime_conf, get_multi_file_processing_id(), - true); + ¶ms->mime_decode_conf, &mime_conf, get_multi_file_processing_id()); // Show file processing the Content-Type header as if it were regular data. // This will enable it to find the boundary string. diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc index 2802d19af..13a1ce932 100755 --- a/src/service_inspectors/http_inspect/test/http_module_test.cc +++ b/src/service_inspectors/http_inspect/test/http_module_test.cc @@ -53,6 +53,10 @@ LiteralSearch::Handle* LiteralSearch::setup() { return nullptr; } void LiteralSearch::cleanup(LiteralSearch::Handle*) {} LiteralSearch* LiteralSearch::instantiate(LiteralSearch::Handle*, const uint8_t*, unsigned, bool, bool) { return nullptr; } +void DecodeConfig::set_decompress_pdf(bool) {} +void DecodeConfig::set_decompress_swf(bool) {} +void DecodeConfig::set_decompress_zip(bool) {} +void DecodeConfig::set_decompress_vba(bool) {} } void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc index 7fcfa6c78..b6823f702 100755 --- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc +++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc @@ -48,6 +48,10 @@ LiteralSearch::Handle* LiteralSearch::setup() { return nullptr; } void LiteralSearch::cleanup(LiteralSearch::Handle*) {} LiteralSearch* LiteralSearch::instantiate(LiteralSearch::Handle*, const uint8_t*, unsigned, bool, bool) { return nullptr; } +void DecodeConfig::set_decompress_pdf(bool) {} +void DecodeConfig::set_decompress_swf(bool) {} +void DecodeConfig::set_decompress_zip(bool) {} +void DecodeConfig::set_decompress_vba(bool) {} } void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { }