Merge pull request #1931 in SNORT/snort3 from ~KATHARVE/snort3:http_mime_file to...

Squashed commit of the following:

commit 159de978a1e07db74de106810e3f268690488567
Author: Katura Harvey <katharve@cisco.com>
Date:   Wed Jan 8 15:50:36 2020 -0500

    mime: support simultaneous file processing of MIME-encoded files over HTTP/1.1

diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc
index 2281877a0e9fe813bac249e7fcbaba716d5a0338..e25a82889f3a6e1481771845b1325a861a25a062 100644 (file)
--- a/src/mime/file_mime_process.cc
+++ b/src/mime/file_mime_process.cc
@@ -28,6 +28,7 @@
 
 #include "detection/detection_engine.h"
 #include "file_api/file_flows.h"
+#include "hash/hashfcn.h"
 #include "log/messages.h"
 #include "search_engines/search_tool.h"
 #include "utils/util_cstring.h"
@@ -597,13 +598,8 @@ const uint8_t* MimeSession::process_mime_data_paf(
         }
 
         /*Process file type/file signature*/
-        FileFlows* file_flows = FileFlows::get_file_flows(flow);
-        if (file_flows && file_flows->file_process(p, buffer, buf_size, position, upload)
-            && (isFileStart(position)) && log_state)
-        {
-            file_flows->set_file_name((const uint8_t*)filename.c_str(), filename.length());
-            filename.clear();
-        }
+        mime_file_process(p, buffer, buf_size, position, upload);
+
         if (mime_stats)
         {
             switch (decode_state->get_decode_type())
@@ -639,12 +635,20 @@ const uint8_t* MimeSession::process_mime_data_paf(
     return end;
 }
 
+void MimeSession::reset_file_data()
+{
+    // Clear MIME's file data to prepare for next file
+    file_counter++;
+    file_process_offset = 0;
+    current_mime_file_id = 0;
+    continue_inspecting_file = true;
+}
+
 // Main function for mime processing
 // This should be called when mime data is available
 const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start,
     int data_size, bool upload, FilePosition position)
 {
-    Flow* flow = p->flow;
     const uint8_t* attach_start = start;
     const uint8_t* attach_end;
 
@@ -652,12 +656,14 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start,
 
     if (position != SNORT_FILE_POSITION_UNKNOWN)
     {
+        if (position == SNORT_FILE_START or position == SNORT_FILE_FULL)
+            reset_file_data();
         process_mime_data_paf(p, attach_start, data_end_marker,
             upload, position);
         return data_end_marker;
     }
 
-    initFilePosition(&position, get_file_processed_size(flow));
+    initFilePosition(&position, file_process_offset);
     /* look for boundary */
     while (start < data_end_marker)
     {
@@ -668,6 +674,7 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start,
             finalFilePosition(&position);
             process_mime_data_paf(p, attach_start, attach_end,
                 upload, position);
+            reset_file_data();
             data_state = STATE_MIME_HEADER;
             position = SNORT_FILE_START;
             attach_start = start + 1;
@@ -678,7 +685,7 @@ const uint8_t* MimeSession::process_mime_data(Packet* p, const uint8_t* start,
 
     if ((start == data_end_marker) && (attach_start < data_end_marker))
     {
-        updateFilePosition(&position, get_file_processed_size(flow));
+        updateFilePosition(&position, file_process_offset);
         process_mime_data_paf(p, attach_start, data_end_marker,
             upload, position);
     }
@@ -800,11 +807,12 @@ void MimeSession::exit()
         delete mime_hdr_search_mpse;
 }
 
-MimeSession::MimeSession(DecodeConfig* dconf, MailLogConfig* lconf)
+MimeSession::MimeSession(DecodeConfig* dconf, MailLogConfig* lconf, uint64_t base_file_id)
 {
     decode_conf = dconf;
     log_config =  lconf;
     log_state = new MailLogState(log_config);
+    session_base_file_id = base_file_id;
     reset_mime_paf_state(&mime_boundary);
 }
 
@@ -817,3 +825,53 @@ MimeSession::~MimeSession()
         delete(log_state);
 }
 
+uint64_t MimeSession::get_mime_file_id()
+{
+    if (!current_mime_file_id)
+    {
+        const int data_len = sizeof(session_base_file_id) + sizeof(file_counter);
+        uint8_t data[data_len];
+        memcpy(data, (void*)&session_base_file_id, sizeof(session_base_file_id));
+        memcpy(data + sizeof(session_base_file_id), (void*)&file_counter, sizeof(file_counter));
+        current_mime_file_id = str_to_hash(data, data_len);
+    }
+    return current_mime_file_id;
+}
+
+void MimeSession::mime_file_process(Packet* p, const uint8_t* data, int data_size,
+    FilePosition position, bool upload)
+{
+    Flow* flow = p->flow;
+    FileFlows* file_flows = FileFlows::get_file_flows(flow);
+    if(!file_flows)
+        return;
+
+    if (continue_inspecting_file or position == SNORT_FILE_END)
+    {
+        if (session_base_file_id)
+        {
+            const FileDirection dir = upload? FILE_UPLOAD : FILE_DOWNLOAD;
+            uint64_t offset = file_process_offset;
+            // MIME has found the end of a file that file processing didn't want - tell file
+            // processing it can clear this file's data
+            if (!continue_inspecting_file)
+                offset = 0;
+            uint64_t file_id = get_mime_file_id();
+            continue_inspecting_file = file_flows->file_process(p, file_id, data, data_size, offset,
+                dir, file_id, position);
+        }
+        else
+        {
+            continue_inspecting_file = file_flows->file_process(p, data, data_size, position,
+                upload);
+        }
+        file_process_offset += data_size;
+        if (continue_inspecting_file and (isFileStart(position)) && log_state)
+        {
+            file_flows->set_file_name((const uint8_t*)filename.c_str(), filename.length());
+            filename.clear();
+        }
+    }
+    if (position == SNORT_FILE_FULL or position == SNORT_FILE_END)
+        reset_file_data();
+}

diff --git a/src/mime/file_mime_process.h b/src/mime/file_mime_process.h
index 549c8151c7c33abd8d7f4e7bcf6aec42cf7bc3b0..682c2ee7d05a124f44d45a8cc4f2630e936ae428 100644 (file)
--- a/src/mime/file_mime_process.h
+++ b/src/mime/file_mime_process.h
@@ -55,7 +55,7 @@ namespace snort
 class SO_PUBLIC MimeSession
 {
 public:
-    MimeSession(DecodeConfig*, MailLogConfig*);
+    MimeSession(DecodeConfig*, MailLogConfig*, uint64_t base_file_id=0);
     virtual ~MimeSession();
 
     MimeSession(const MimeSession&) = delete;
@@ -84,6 +84,16 @@ private:
     MailLogState* log_state = nullptr;
     MimeStats* mime_stats = nullptr;
     std::string filename;
+    bool continue_inspecting_file = true;
+    // This counter is not an accurate count of files; used only for creating a unique mime_file_id
+    uint32_t file_counter = 0;
+    uint32_t file_process_offset = 0;
+    uint64_t session_base_file_id = 0;
+    uint64_t current_mime_file_id = 0;
+    uint64_t get_mime_file_id();
+    void mime_file_process(Packet* p, const uint8_t* data, int data_size,
+        FilePosition position, bool upload);
+    void reset_file_data();
 
     // SMTP, IMAP, POP might have different implementation for this
     virtual int handle_header_line(const uint8_t*, const uint8_t*, int, Packet*) { return 0; }

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 1bcd3bf3e32124aa6bb24b267a9329c48546af41..6cdeb594d3004bea66a66f315eeb60d336a1c7af 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -329,7 +329,8 @@ void HttpMsgHeader::setup_file_processing()
         {
             if (boundary_present(content_type))
             {
-                session_data->mime_state[source_id] = new MimeSession(&decode_conf, &mime_conf);
+                session_data->mime_state[source_id] = new MimeSession(&decode_conf, &mime_conf,
+                    transaction->get_file_processing_id(source_id));
                 // Show file processing the Content-Type header as if it were regular data.
                 // This will enable it to find the boundary string.
                 // FIXIT-L develop a proper interface for passing the boundary string.

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
index 007ecccc449fa3a102fba89de11209741c1db268..698b2baa83fc23f9b684fc18c324b85d40bf5b37 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
@@ -102,6 +102,8 @@ bool HttpStreamSplitter::finish(Flow* flow)
         return true;
     }
 
+    // FIXIT-M No longer necessary to send an empty body section because the header section is
+    // always forwarded to detection.
     // If the message has been truncated immediately following the start line or immediately
     // following the headers (a body was expected) then we need to process an empty section to
     // provide an inspection section. Otherwise the start line and headers won't go through
@@ -130,10 +132,12 @@ bool HttpStreamSplitter::finish(Flow* flow)
         if (!session_data->mime_state[source_id])
         {
             FileFlows* file_flows = FileFlows::get_file_flows(flow);
-            const bool download = (source_id == SRC_SERVER);
+            const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD;
 
             size_t file_index = 0;
+            uint64_t file_processing_id = 0;
 
+            // FIXIT-L How can there be a file in progress and no transaction in this direction?
             if (session_data->transaction[source_id] != nullptr)
             {
                 HttpMsgRequest* request = session_data->transaction[source_id]->get_request();
@@ -141,9 +145,11 @@ bool HttpStreamSplitter::finish(Flow* flow)
                 {
                     file_index = request->get_http_uri()->get_file_proc_hash();
                 }
+                file_processing_id =
+                    session_data->transaction[source_id]->get_file_processing_id(source_id);
             }
-
-            file_flows->file_process(packet, nullptr, 0, SNORT_FILE_END, !download, file_index);
+            file_flows->file_process(packet, file_index, nullptr, 0, 0, dir, file_processing_id,
+                SNORT_FILE_END);
         }
         else
         {