Pull request #3306: http_inspect: do file decompression and utf decoding on non-MIME...

Merge in SNORT/snort3 from ~KATHARVE/snort3:non_mime_uploads to master

Squashed commit of the following:

commit 5af71a0295291bafdd017fa9468a016ed0dd2cb8
Author: Katura Harvey <katharve@cisco.com>
Date:   Fri Mar 11 13:52:10 2022 -0500

    http_inspect: do file decompression and utf decoding on non-MIME uploads

diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc
index 9b93710537c1933210004b23dfaf2ddcb4428249..117d1b1381f85c91a02f7a03ffcc86f656deec13 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.cc
+++ b/src/service_inspectors/http_inspect/http_flow_data.cc
@@ -123,15 +123,12 @@ HttpFlowData::~HttpFlowData()
             inflateEnd(compress_stream[k]);
             delete compress_stream[k];
         }
-        if (mime_state[k] != nullptr)
-        {
-            delete mime_state[k];
-        }
+        delete mime_state[k];
+        delete utf_state[k];
+        if (fd_state[k] != nullptr)
+            File_Decomp_StopFree(fd_state[k]);
     }
 
-    delete utf_state;
-    if (fd_state != nullptr)
-        File_Decomp_StopFree(fd_state);
     delete_pipeline();
 
     while (discard_list != nullptr)
@@ -168,10 +165,14 @@ void HttpFlowData::half_reset(SourceId source_id)
         delete compress_stream[source_id];
         compress_stream[source_id] = nullptr;
     }
-    if (mime_state[source_id] != nullptr)
+    delete mime_state[source_id];
+    mime_state[source_id] = nullptr;
+    delete utf_state[source_id];
+    utf_state[source_id] = nullptr;
+    if (fd_state[source_id] != nullptr)
     {
-        delete mime_state[source_id];
-        mime_state[source_id] = nullptr;
+        File_Decomp_StopFree(fd_state[source_id]);
+        fd_state[source_id] = nullptr;
     }
     delete infractions[source_id];
     infractions[source_id] = new HttpInfractions;
@@ -191,13 +192,6 @@ void HttpFlowData::half_reset(SourceId source_id)
         if (transaction[SRC_SERVER]->final_response())
             expected_trans_num[SRC_SERVER]++;
         status_code_num = STAT_NOT_PRESENT;
-        delete utf_state;
-        utf_state = nullptr;
-        if (fd_state != nullptr)
-        {
-            File_Decomp_StopFree(fd_state);
-            fd_state = nullptr;
-        }
     }
 }
 
@@ -398,8 +392,10 @@ void HttpFlowData::show(FILE* out_file) const
         pipeline_back, pipeline_overflow, pipeline_underflow);
     fprintf(out_file, "Cutter: %s/%s\n", (cutter[0] != nullptr) ? "Present" : "nullptr",
         (cutter[1] != nullptr) ? "Present" : "nullptr");
-    fprintf(out_file, "utf_state: %s\n", (utf_state != nullptr) ? "Present" : "nullptr");
-    fprintf(out_file, "fd_state: %s\n", (fd_state != nullptr) ? "Present" : "nullptr");
+    fprintf(out_file, "utf_state: %s/%s\n", (utf_state[0] != nullptr) ? "Present" : "nullptr",
+        (utf_state[1] != nullptr) ? "Present" : "nullptr");
+    fprintf(out_file, "fd_state: %s/%s\n", (fd_state[0] != nullptr) ? "Present" : "nullptr",
+        (fd_state[1] != nullptr) ? "Present" : "nullptr");
     fprintf(out_file, "mime_state: %s/%s\n", (mime_state[0] != nullptr) ? "Present" : "nullptr",
         (mime_state[1] != nullptr) ? "Present" : "nullptr");
 }

diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index 1a5d21d85ed6d5006be27be9bc82bb7721d713a6..e25f2de3da4b1a7c8bcdc2b03b50bc62339de056 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -160,10 +160,10 @@ private:
         HttpInfractions* infractions = nullptr;
         HttpEventGen* events = nullptr;
     };
-    FdCallbackContext fd_alert_context; // SRC_SERVER only
+    FdCallbackContext fd_alert_context[2];
     snort::MimeSession* mime_state[2] = { nullptr, nullptr };
-    snort::UtfDecodeSession* utf_state = nullptr; // SRC_SERVER only
-    fd_session_t* fd_state = nullptr; // SRC_SERVER only
+    snort::UtfDecodeSession* utf_state[2] = { nullptr, nullptr };
+    fd_session_t* fd_state[2] = { nullptr, nullptr };
     int64_t file_depth_remaining[2] = { HttpCommon::STAT_NOT_PRESENT,
         HttpCommon::STAT_NOT_PRESENT };
     int64_t detect_depth_remaining[2] = { HttpCommon::STAT_NOT_PRESENT,

diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index ed5bc524844373a69a2b99c61dad06c6f1988d0c..2c7e3156eb12ae344dda1c6783f87f9806ea8806 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -227,18 +227,19 @@ void HttpMsgBody::analyze()
 
 void HttpMsgBody::do_utf_decoding(const Field& input, Field& output)
 {
-    if ((source_id == SRC_CLIENT) || (session_data->utf_state == nullptr) || (input.length() == 0))
+    if ((session_data->mime_state[source_id] != nullptr) ||
+        (session_data->utf_state[source_id] == nullptr) || (input.length() == 0))
     {
         output.set(input);
         return;
     }
 
-    if (session_data->utf_state->is_utf_encoding_present())
+    if (session_data->utf_state[source_id]->is_utf_encoding_present())
     {
         int bytes_copied;
         bool decoded;
         uint8_t* buffer = new uint8_t[input.length()];
-        decoded = session_data->utf_state->decode_utf(
+        decoded = session_data->utf_state[source_id]->decode_utf(
             input.start(), input.length(), buffer, input.length(), &bytes_copied);
 
         if (!decoded)
@@ -268,53 +269,55 @@ void HttpMsgBody::get_ole_data()
     uint8_t* ole_data_ptr;
     uint32_t ole_len;
 
-    session_data->fd_state->get_ole_data(ole_data_ptr, ole_len);
+    session_data->fd_state[source_id]->get_ole_data(ole_data_ptr, ole_len);
 
     if (ole_data_ptr)
     {
         ole_data.set(ole_len, ole_data_ptr, false);
 
         //Reset the ole data ptr once it is stored in msg body
-        session_data->fd_state->ole_data_reset();
+        session_data->fd_state[source_id]->ole_data_reset();
     }
 }
     
 void HttpMsgBody::do_file_decompression(const Field& input, Field& output)
 {
-    if ((source_id == SRC_CLIENT) || (session_data->fd_state == nullptr))
+    if ((session_data->mime_state[source_id] != nullptr) ||
+        session_data->fd_state[source_id] == nullptr)
     {
         output.set(input);
         return;
     }
     const uint32_t buffer_size = session_data->file_decomp_buffer_size_remaining[source_id];
     uint8_t* buffer = new uint8_t[buffer_size];
-    session_data->fd_alert_context.infractions = transaction->get_infractions(source_id);
-    session_data->fd_alert_context.events = session_data->events[source_id];
-    session_data->fd_state->Next_In = input.start();
-    session_data->fd_state->Avail_In = (uint32_t)input.length();
-    session_data->fd_state->Next_Out = buffer;
-    session_data->fd_state->Avail_Out = buffer_size;
+    session_data->fd_alert_context[source_id].infractions = transaction->get_infractions(source_id);
+    session_data->fd_alert_context[source_id].events = session_data->events[source_id];
+    session_data->fd_state[source_id]->Next_In = input.start();
+    session_data->fd_state[source_id]->Avail_In = (uint32_t)input.length();
+    session_data->fd_state[source_id]->Next_Out = buffer;
+    session_data->fd_state[source_id]->Avail_Out = buffer_size;
 
-    const fd_status_t status = File_Decomp(session_data->fd_state);
+    const fd_status_t status = File_Decomp(session_data->fd_state[source_id]);
 
     switch(status)
     {
     case File_Decomp_DecompError:
-        File_Decomp_Alert(session_data->fd_state, session_data->fd_state->Error_Event);
+        File_Decomp_Alert(session_data->fd_state[source_id],
+            session_data->fd_state[source_id]->Error_Event);
         // Fall through
     case File_Decomp_NoSig:
     case File_Decomp_Error:
         delete[] buffer;
         output.set(input);
-        File_Decomp_StopFree(session_data->fd_state);
-        session_data->fd_state = nullptr;
+        File_Decomp_StopFree(session_data->fd_state[source_id]);
+        session_data->fd_state[source_id] = nullptr;
         break;
     case File_Decomp_BlockOut:
         add_infraction(INF_FILE_DECOMPR_OVERRUN);
         create_event(EVENT_FILE_DECOMPR_OVERRUN);
         // Fall through
     default:
-        const uint32_t output_length = session_data->fd_state->Next_Out - buffer;
+        const uint32_t output_length = session_data->fd_state[source_id]->Next_Out - buffer;
         output.set(output_length, buffer, true);
         assert((uint64_t)session_data->file_decomp_buffer_size_remaining[source_id] >=
             output_length);

diff --git a/src/service_inspectors/http_inspect/http_msg_body_chunk.cc b/src/service_inspectors/http_inspect/http_msg_body_chunk.cc
index 7c91250b7f2ea7569883d3369aa645fd325d68a2..a6992080ed2068c091c863b78b4b77cc5b9b900c 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body_chunk.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body_chunk.cc
@@ -40,10 +40,10 @@ void HttpMsgBodyChunk::update_flow()
             session_data->mime_state[source_id] = nullptr;
         }
 
-        if ((source_id == SRC_SERVER) && (session_data->utf_state != nullptr))
+        if ((source_id == SRC_SERVER) && (session_data->utf_state[source_id] != nullptr))
         {
-            delete session_data->utf_state;
-            session_data->utf_state = nullptr;
+            delete session_data->utf_state[source_id];
+            session_data->utf_state[source_id] = nullptr;
         }
     }
     else

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 115168a0dacc2dac13d157b8975fadae0298a984..c0c7687b130428c2cff308d3b6f3023c25b86095 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -608,7 +608,7 @@ void HttpMsgHeader::setup_encoding_decompression()
 
 void HttpMsgHeader::setup_utf_decoding()
 {
-    if (!params->normalize_utf || source_id == SRC_CLIENT )
+    if (!params->normalize_utf || session_data->mime_state[source_id] )
         return;
 
     Field last_token;
@@ -651,28 +651,28 @@ void HttpMsgHeader::setup_utf_decoding()
         }
     }
 
-    session_data->utf_state = new UtfDecodeSession();
-    session_data->utf_state->set_decode_utf_state_charset(charset_code);
+    session_data->utf_state[source_id] = new UtfDecodeSession();
+    session_data->utf_state[source_id]->set_decode_utf_state_charset(charset_code);
 }
 
 void HttpMsgHeader::setup_file_decompression()
 {
-    if (source_id == SRC_CLIENT ||
-       (!params->decompress_pdf && !params->decompress_swf && !params->decompress_zip))
+    if (session_data->mime_state[source_id] ||
+        (!params->decompress_pdf && !params->decompress_swf && !params->decompress_zip))
         return;
 
-    session_data->fd_state = File_Decomp_New();
-    session_data->fd_state->Modes =
+    session_data->fd_state[source_id] = File_Decomp_New();
+    session_data->fd_state[source_id]->Modes =
         (params->decompress_pdf ? FILE_PDF_DEFL_BIT : 0) |
         (params->decompress_swf ? (FILE_SWF_ZLIB_BIT | FILE_SWF_LZMA_BIT) : 0) |
         (params->decompress_zip ? FILE_ZIP_DEFL_BIT : 0) |
         (params->decompress_vba ? FILE_VBA_EXTR_BIT : 0);
-    session_data->fd_state->Alert_Callback = HttpMsgBody::fd_event_callback;
-    session_data->fd_state->Alert_Context = &session_data->fd_alert_context;
-    session_data->fd_state->Compr_Depth = 0;
-    session_data->fd_state->Decompr_Depth = 0;
+    session_data->fd_state[source_id]->Alert_Callback = HttpMsgBody::fd_event_callback;
+    session_data->fd_state[source_id]->Alert_Context = &session_data->fd_alert_context[source_id];
+    session_data->fd_state[source_id]->Compr_Depth = 0;
+    session_data->fd_state[source_id]->Decompr_Depth = 0;
 
-    (void)File_Decomp_Init(session_data->fd_state);
+    (void)File_Decomp_Init(session_data->fd_state[source_id]);
 }
 
 // Each file processed has a unique id per flow: hash(source_id, transaction_id, h2_stream_id)