Merge pull request #2376 in SNORT/snort3 from ~THOPETER/snort3:nhttp147 to master

Squashed commit of the following:

commit 065b7738aaf7712fba8275f0cd83519bd79a232d
Author: Tom Peters <thopeter@cisco.com>
Date:   Mon Aug 3 15:05:23 2020 -0400

    http_inspect: test tool enhancement

diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc
index bedc35db31493a6e1b368b389b832bda5fda81bc..61788939151a38448434f413dee16bb614a8bc0c 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.cc
+++ b/src/service_inspectors/http_inspect/http_flow_data.cc
@@ -120,6 +120,7 @@ void HttpFlowData::half_reset(SourceId source_id)
     version_id[source_id] = VERS__NOT_PRESENT;
     data_length[source_id] = STAT_NOT_PRESENT;
     body_octets[source_id] = STAT_NOT_PRESENT;
+    file_octets[source_id] = STAT_NOT_PRESENT;
     partial_inspected_octets[source_id] = 0;
     section_size_target[source_id] = 0;
     stretch_section_to_packet[source_id] = false;
@@ -265,6 +266,7 @@ void HttpFlowData::show(FILE* out_file) const
     fprintf(out_file, "File depth remaining: %" PRIi64 "/%" PRIi64 "\n", file_depth_remaining[0],
         file_depth_remaining[1]);
     fprintf(out_file, "Body octets: %" PRIi64 "/%" PRIi64 "\n", body_octets[0], body_octets[1]);
+    fprintf(out_file, "File octets: %" PRIi64 "/%" PRIi64 "\n", file_octets[0], file_octets[1]);
     fprintf(out_file, "Pipelining: front %d back %d overflow %d underflow %d\n", pipeline_front,
         pipeline_back, pipeline_overflow, pipeline_underflow);
     fprintf(out_file, "Cutter: %s/%s\n", (cutter[0] != nullptr) ? "Present" : "nullptr",

diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index 43a47e6ddb20bc1a4b0196435256c6b8b9519b05..b1a8ff3d42504e10719baee1f50296abf3fecb04 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -161,6 +161,8 @@ private:
 
     // number of user data octets seen so far (regular body or chunks)
     int64_t body_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
+    // normalized octets forwarded to file or MIME processing
+    int64_t file_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
     uint32_t partial_inspected_octets[2] = { 0, 0 };
     uint8_t* partial_detect_buffer[2] = { nullptr, nullptr };
     uint32_t partial_detect_length[2] = { 0, 0 };

diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc
index 8082399fcd36d8cc1c2f7d5cb76f211df71aeab4..a5034ae53547630afe8ad54dc2519e11808cb375 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_body.cc
+++ b/src/service_inspectors/http_inspect/http_msg_body.cc
@@ -279,7 +279,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
         }
 
         if (file_flows->file_process(p, file_index, file_data.start(), fp_length,
-            body_octets + session_data->partial_inspected_octets[source_id], dir,
+            session_data->file_octets[source_id], dir,
             transaction->get_file_processing_id(source_id), file_position))
         {
             session_data->file_depth_remaining[source_id] -= fp_length;
@@ -303,11 +303,15 @@ void HttpMsgBody::do_file_processing(const Field& file_data)
             // file processing doesn't want any more data
             session_data->file_depth_remaining[source_id] = 0;
         }
+        session_data->file_octets[source_id] += fp_length;
     }
     else
     {
+        // FIXIT-M this interface does not convey any indication of end of message body. If the
+        // message body ends in the middle of a MIME message the partial file will not be flushed.
         session_data->mime_state[source_id]->process_mime_data(p, file_data.start(),
             file_data.length(), true, SNORT_FILE_POSITION_UNKNOWN);
+        session_data->file_octets[source_id] += file_data.length();
     }
 }
 

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 9161162363fe844e9bdc92b536af920b60e0ba20..d0ce5063eff538f086794f5b6922537c8970ae8a 100644 (file)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -401,6 +401,8 @@ void HttpMsgHeader::prepare_body()
 
 void HttpMsgHeader::setup_file_processing()
 {
+    session_data->file_octets[source_id] = 0;
+
     const int64_t max_file_depth = FileService::get_max_file_depth();
     if (max_file_depth <= 0)
     {

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
index a5ca4eae9d34c03060e7b0c64e5cb07889a6dcd3..e26451dbb05a8b27d70de56766ca853cfb8e6579 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
@@ -125,31 +125,44 @@ bool HttpStreamSplitter::finish(Flow* flow)
             if (!file_flows)
                 return false;
 
-            const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD;
+            const FileDirection dir = (source_id == SRC_SERVER) ? FILE_DOWNLOAD : FILE_UPLOAD;
 
             size_t file_index = 0;
-            uint64_t file_processing_id = 0;
 
-            // FIXIT-L How can there be a file in progress and no transaction in this direction?
-            if (session_data->transaction[source_id] != nullptr)
+            assert(session_data->transaction[source_id] != nullptr);
+            HttpMsgRequest* request = session_data->transaction[source_id]->get_request();
+            if ((request != nullptr) and (request->get_http_uri() != nullptr))
             {
-                HttpMsgRequest* request = session_data->transaction[source_id]->get_request();
-                if ((request != nullptr) and (request->get_http_uri() != nullptr))
-                {
-                    file_index = request->get_http_uri()->get_file_proc_hash();
-                }
-                file_processing_id =
-                    session_data->transaction[source_id]->get_file_processing_id(source_id);
+                file_index = request->get_http_uri()->get_file_proc_hash();
             }
+            const uint64_t file_processing_id =
+                session_data->transaction[source_id]->get_file_processing_id(source_id);
             file_flows->file_process(packet, file_index, nullptr, 0, 0, dir, file_processing_id,
                 SNORT_FILE_END);
+#ifdef REG_TEST
+            if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+            {
+                fprintf(HttpTestManager::get_output_file(),
+                    "File processing finalization during finish()\n");
+                fflush(HttpTestManager::get_output_file());
+            }
+#endif
         }
         else
         {
+            // FIXIT-M The following call does not actually accomplish anything. The MIME interface
+            // needs to be enhanced so that we can communicate end-of-data without side effects.
             session_data->mime_state[source_id]->process_mime_data(packet, nullptr, 0, true,
                 SNORT_FILE_POSITION_UNKNOWN);
             delete session_data->mime_state[source_id];
             session_data->mime_state[source_id] = nullptr;
+#ifdef REG_TEST
+            if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+            {
+                fprintf(HttpTestManager::get_output_file(), "MIME finalization during finish()\n");
+                fflush(HttpTestManager::get_output_file());
+            }
+#endif
         }
         return false;
     }