Pull request #3396: Http mime depth

Merge in SNORT/snort3 from ~KATHARVE/snort3:http_mime_depth to master

Squashed commit of the following:

commit 0a8379db6e1fa866ddc327409324e2ef094f0fa1
Author: Katura Harvey <katharve@cisco.com>
Date:   Wed Apr 20 12:21:33 2022 -0400

    http_inspect: move mime processing outside of file and detect depth

diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc
index 51c78e091956a6075c79ae7ce34b7027f32fc93b..e2ca27238fe225516d59352bab04985bc552114e 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_inspect.cc
+++ b/src/service_inspectors/http_inspect/http_inspect.cc
@@ -132,9 +132,17 @@ HttpInspect::HttpInspect(const HttpParaList* params_) :
     }
 }
 
+HttpInspect::~HttpInspect()
+{
+    delete params->mime_decode_conf;
+    delete params;
+    delete script_finder;
+}
+
 bool HttpInspect::configure(SnortConfig* )
 {
     params->js_norm_param.js_norm->configure();
+    params->mime_decode_conf->sync_all_depths();
 
     return true;
 }

diff --git a/src/service_inspectors/http_inspect/http_inspect.h b/src/service_inspectors/http_inspect/http_inspect.h
index 436a522a18894fbb678734731d4e86bd4f6cf9e3..c6954b36878ee831a417bc959ce0c2cb501684dd 100644 (file)
--- a/src/service_inspectors/http_inspect/http_inspect.h
+++ b/src/service_inspectors/http_inspect/http_inspect.h
@@ -43,7 +43,7 @@ class HttpInspect : public snort::Inspector
 {
 public:
     HttpInspect(const HttpParaList* params_);
-    ~HttpInspect() override { delete params; delete script_finder; }
+    ~HttpInspect() override;
 
     bool get_buf(snort::InspectionBuffer::Type ibt, snort::Packet* p,
         snort::InspectionBuffer& b) override;

diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc
index 733322801b016675181108d3e947bac7928c3f04..6c224dfe9900cc2368682f3e720f389002801c72 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_module.cc
+++ b/src/service_inspectors/http_inspect/http_module.cc
@@ -248,22 +248,18 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*)
     else if (val.is("decompress_pdf"))
     {
         params->decompress_pdf = val.get_bool();
-        params->mime_decode_conf.set_decompress_pdf(val.get_bool());
     }
     else if (val.is("decompress_swf"))
     {
         params->decompress_swf = val.get_bool();
-        params->mime_decode_conf.set_decompress_swf(val.get_bool());
     }
     else if (val.is("decompress_zip"))
     {
         params->decompress_zip = val.get_bool();
-        params->mime_decode_conf.set_decompress_zip(val.get_bool());
     }
     else if (val.is("decompress_vba"))
     {
         params->decompress_vba = val.get_bool();
-        params->mime_decode_conf.set_decompress_vba(val.get_bool());
     }
     else if (val.is("script_detection"))
     {
@@ -486,6 +482,13 @@ bool HttpModule::end(const char* fqn, int, SnortConfig*)
     params->script_detection_handle = script_detection_handle;
 
     prepare_http_header_list(params);
+
+    params->mime_decode_conf = new DecodeConfig();
+    params->mime_decode_conf->set_decompress_pdf(params->decompress_pdf);
+    params->mime_decode_conf->set_decompress_swf(params->decompress_swf);
+    params->mime_decode_conf->set_decompress_zip(params->decompress_zip);
+    params->mime_decode_conf->set_decompress_vba(params->decompress_vba);
+
     return true;
 }
 

diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h
index ced0031336043e4a5c076748339ea1fe02b84ed2..77063b916cfd2977d95b101726f429ce6b8552f5 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_module.h
+++ b/src/service_inspectors/http_inspect/http_module.h
@@ -58,7 +58,7 @@ public:
     bool decompress_swf = false;
     bool decompress_zip = false;
     bool decompress_vba = false;
-    snort::DecodeConfig mime_decode_conf;
+    snort::DecodeConfig* mime_decode_conf;
     bool script_detection = false;
     snort::LiteralSearch::Handle* script_detection_handle = nullptr;
     bool publish_request_body = true;

diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index dd44d0460632106ae8c3d27d4c44ebae8e3d7dba..1515302578b8a9572a62b10ee538cd3d8996e837 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -142,8 +142,32 @@ void HttpMsgBody::analyze()
             msg_text_new.length() : pub_depth_remaining;
         pub_depth_remaining -= publish_length;
     }
+    
+    if (session_data->mime_state[source_id])
+    {
+        // FIXIT-M this interface does not convey any indication of end of message body. If the
+        // message body ends in the middle of a MIME message the partial file will not be flushed.
 
-    if (session_data->file_depth_remaining[source_id] > 0 or
+        Packet* p = DetectionEngine::get_current_packet();
+        const uint8_t* const section_end = msg_text_new.start() + msg_text_new.length();
+        const uint8_t* ptr = msg_text_new.start();
+        while (ptr < section_end)
+        {
+            // After process_mime_data(), ptr will point to the last byte processed in the current
+            // MIME part
+            ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr,
+                (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN);
+            ptr++;
+        }
+        
+        const BufferData& vba_buf = session_data->mime_state[source_id]->get_ole_buf();
+        if (vba_buf.data_ptr())
+            ole_data.set(vba_buf.length(), vba_buf.data_ptr());
+
+        detect_data.set(msg_text.length(), msg_text.start());
+    }
+
+    else if (session_data->file_depth_remaining[source_id] > 0 or
         session_data->detect_depth_remaining[source_id] > 0)
     {
         do_utf_decoding(msg_text_new, decoded_body);
@@ -217,9 +241,8 @@ void HttpMsgBody::analyze()
             // file attachment body data.
             // FIXIT-E currently the file_data buffer is set to the body of the last attachment per
             // message section.
-            if (!session_data->mime_state[source_id])
-                set_file_data(const_cast<uint8_t*>(detect_data.start()),
-                    (unsigned)detect_data.length());
+            set_file_data(const_cast<uint8_t*>(detect_data.start()),
+                (unsigned)detect_data.length());
         }
     }
     body_octets += msg_text.length();
@@ -228,8 +251,7 @@ void HttpMsgBody::analyze()
 
 void HttpMsgBody::do_utf_decoding(const Field& input, Field& output)
 {
-    if ((session_data->mime_state[source_id] != nullptr) ||
-        (session_data->utf_state[source_id] == nullptr) || (input.length() == 0))
+    if ((session_data->utf_state[source_id] == nullptr) || (input.length() == 0))
     {
         output.set(input);
         return;
@@ -283,8 +305,7 @@ void HttpMsgBody::get_ole_data()
     
 void HttpMsgBody::do_file_decompression(const Field& input, Field& output)
 {
-    if ((session_data->mime_state[source_id] != nullptr) ||
-        session_data->fd_state[source_id] == nullptr)
+    if (session_data->fd_state[source_id] == nullptr)
     {
         output.set(input);
         return;
@@ -422,74 +443,49 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
         return;
     }
 
-    if (!session_data->mime_state[source_id])
-    {
-        const int32_t fp_length = (file_data.length() <=
-            session_data->file_depth_remaining[source_id]) ?
-            file_data.length() : session_data->file_depth_remaining[source_id];
+    const int32_t fp_length = (file_data.length() <=
+        session_data->file_depth_remaining[source_id]) ?
+        file_data.length() : session_data->file_depth_remaining[source_id];
 
-        FileFlows* file_flows = FileFlows::get_file_flows(flow);
-        if (!file_flows)
-            return;
+    FileFlows* file_flows = FileFlows::get_file_flows(flow);
+    if (!file_flows)
+        return;
 
-        const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD;
+    const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD;
 
-        const uint64_t file_index = get_header(source_id)->get_file_cache_index();
+    const uint64_t file_index = get_header(source_id)->get_file_cache_index();
 
-        bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(),
-            fp_length, session_data->file_octets[source_id], dir,
-            get_header(source_id)->get_multi_file_processing_id(), file_position);
-        if (continue_processing_file)
-        {
-            session_data->file_depth_remaining[source_id] -= fp_length;
+    bool continue_processing_file = file_flows->file_process(p, file_index, file_data.start(),
+        fp_length, session_data->file_octets[source_id], dir,
+        get_header(source_id)->get_multi_file_processing_id(), file_position);
+    if (continue_processing_file)
+    {
+        session_data->file_depth_remaining[source_id] -= fp_length;
 
-            // With the first piece of the file we must provide the filename and URI
-            if (front)
+        // With the first piece of the file we must provide the filename and URI
+        if (front)
+        {
+            if (request != nullptr)
             {
-                if (request != nullptr)
-                {
-                    const uint8_t* filename_buffer;
-                    const uint8_t* uri_buffer;
-                    uint32_t filename_length;
-                    uint32_t uri_length;
-                    get_file_info(dir, filename_buffer, filename_length, uri_buffer, uri_length);
-
-                    continue_processing_file = file_flows->set_file_name(filename_buffer,
-                        filename_length, 0,
-                        get_header(source_id)->get_multi_file_processing_id(), uri_buffer,
-                        uri_length);
-                }
+                const uint8_t* filename_buffer;
+                const uint8_t* uri_buffer;
+                uint32_t filename_length;
+                uint32_t uri_length;
+                get_file_info(dir, filename_buffer, filename_length, uri_buffer, uri_length);
+
+                continue_processing_file = file_flows->set_file_name(filename_buffer,
+                    filename_length, 0,
+                    get_header(source_id)->get_multi_file_processing_id(), uri_buffer,
+                    uri_length);
             }
         }
-        if (!continue_processing_file)
-        {
-            // file processing doesn't want any more data
-            session_data->file_depth_remaining[source_id] = 0;
-        }
-        session_data->file_octets[source_id] += fp_length;
     }
-    else
+    if (!continue_processing_file)
     {
-        // FIXIT-M this interface does not convey any indication of end of message body. If the
-        // message body ends in the middle of a MIME message the partial file will not be flushed.
-
-        const uint8_t* const section_end = file_data.start() + file_data.length();
-        const uint8_t* ptr = file_data.start();
-        while (ptr < section_end)
-        {
-            // After process_mime_data(), ptr will point to the last byte processed in the current
-            // MIME part
-            ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr,
-                (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN);
-            ptr++;
-        }
-        
-        const BufferData& vba_buf = session_data->mime_state[source_id]->get_ole_buf();
-        if (vba_buf.data_ptr())
-            ole_data.set(vba_buf.length(), vba_buf.data_ptr());
-        
-        session_data->file_octets[source_id] += file_data.length();
+        // file processing doesn't want any more data
+        session_data->file_depth_remaining[source_id] = 0;
     }
+    session_data->file_octets[source_id] += fp_length;
 }
 
 // Parses out the filename and URI associated with this file.

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index c0c7687b130428c2cff308d3b6f3023c25b86095..3e9de469977e001bbd2343e8044f4d519a169d2f 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -443,15 +443,26 @@ void HttpMsgHeader::update_flow()
     }
 }
 
-// Common activities of preparing for upcoming regular body or chunked body
+// Common activities of preparing for upcoming body
 void HttpMsgHeader::prepare_body()
 {
     session_data->body_octets[source_id] = 0;
-    const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth :
-        params->response_depth;
-    session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX;
-    params->js_norm_param.js_norm->set_detection_depth(session_data->detect_depth_remaining[source_id]);
-
+    setup_mime();
+    if (!session_data->mime_state[source_id])
+    {
+        const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth :
+            params->response_depth;
+        session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX;
+        params->js_norm_param.js_norm->set_detection_depth(session_data->detect_depth_remaining[source_id]);
+    }
+    else
+    {
+        // File and decode depths are per attachment, so if either is greater than 0 we inspect the
+        // full message body. Currently the decode depths are not configurable for http_inspect so
+        // are always the default of unlimited, meaning for MIME we always inspect the full message
+        // body
+        session_data->detect_depth_remaining[source_id] = INT64_MAX;
+    }
     if ((source_id == SRC_CLIENT) and params->publish_request_body and session_data->for_http2)
     {
         session_data->publish_octets[source_id] = 0;
@@ -479,20 +490,8 @@ void HttpMsgHeader::prepare_body()
     }
 }
 
-void HttpMsgHeader::setup_file_processing()
+void HttpMsgHeader::setup_mime()
 {
-    session_data->file_octets[source_id] = 0;
-
-    const int64_t max_file_depth = FileService::get_max_file_depth();
-    if (max_file_depth <= 0)
-    {
-        session_data->file_depth_remaining[source_id] = 0;
-        return;
-    }
-
-    // Generate the unique file id for multi file processing
-    set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id());
-
     // Do we meet all the conditions for MIME file processing?
     if (source_id == SRC_CLIENT)
     {
@@ -501,15 +500,18 @@ void HttpMsgHeader::setup_file_processing()
         {
             if (boundary_present(content_type))
             {
+                // Generate the unique file id for multi file processing
+                set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id());
+
                 Packet* p = DetectionEngine::get_current_packet();
                 const Field& uri = request->get_uri_norm_classic();
                 if (uri.length() > 0)
                     session_data->mime_state[source_id] = new MimeSession(p,
-                        &params->mime_decode_conf, &mime_conf, get_multi_file_processing_id(),
+                        params->mime_decode_conf, &mime_conf, get_multi_file_processing_id(),
                         uri.start(), uri.length());
                 else
                     session_data->mime_state[source_id] = new MimeSession(p,
-                        &params->mime_decode_conf, &mime_conf, get_multi_file_processing_id());
+                        params->mime_decode_conf, &mime_conf, get_multi_file_processing_id());
 
                 // Show file processing the Content-Type header as if it were regular data.
                 // This will enable it to find the boundary string.
@@ -525,15 +527,28 @@ void HttpMsgHeader::setup_file_processing()
             }
         }
     }
+}
 
-    // Otherwise do regular file processing
-    if (session_data->mime_state[source_id] == nullptr)
+void HttpMsgHeader::setup_file_processing()
+{
+    if (session_data->mime_state[source_id])
+        return;
+
+    session_data->file_octets[source_id] = 0;
+    const int64_t max_file_depth = FileService::get_max_file_depth();
+    if (max_file_depth <= 0)
     {
-        session_data->file_depth_remaining[source_id] = max_file_depth;
-        FileFlows* file_flows = FileFlows::get_file_flows(flow);
-        if (!file_flows)
-            session_data->file_depth_remaining[source_id] = 0;
+        session_data->file_depth_remaining[source_id] = 0;
+        return;
     }
+
+    // Generate the unique file id for multi file processing
+    set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id());
+
+    session_data->file_depth_remaining[source_id] = max_file_depth;
+    FileFlows* file_flows = FileFlows::get_file_flows(flow);
+    if (!file_flows)
+        session_data->file_depth_remaining[source_id] = 0;
 }
 
 void HttpMsgHeader::setup_encoding_decompression()

diff --git a/src/service_inspectors/http_inspect/http_msg_header.h b/src/service_inspectors/http_inspect/http_msg_header.h
index 64df1329e6a33adc6cac2ebe738e61d6cf1b661e..06f9fc3d383091bb31eda6f8ed109cb77247a494 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_header.h
+++ b/src/service_inspectors/http_inspect/http_msg_header.h
@@ -53,6 +53,7 @@ public:
 
 private:
     void prepare_body();
+    void setup_mime();
     void setup_file_processing();
     void setup_encoding_decompression();
     void setup_utf_decoding();