From: Mike Stepanek (mstepane) Date: Wed, 5 Aug 2020 13:59:07 +0000 (+0000) Subject: Merge pull request #2376 in SNORT/snort3 from ~THOPETER/snort3:nhttp147 to master X-Git-Tag: 3.0.2-4~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7a46b4106d2c9a844043c42397643a08a8aba489;p=thirdparty%2Fsnort3.git Merge pull request #2376 in SNORT/snort3 from ~THOPETER/snort3:nhttp147 to master Squashed commit of the following: commit 065b7738aaf7712fba8275f0cd83519bd79a232d Author: Tom Peters Date: Mon Aug 3 15:05:23 2020 -0400 http_inspect: test tool enhancement --- diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index bedc35db3..617889391 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -120,6 +120,7 @@ void HttpFlowData::half_reset(SourceId source_id) version_id[source_id] = VERS__NOT_PRESENT; data_length[source_id] = STAT_NOT_PRESENT; body_octets[source_id] = STAT_NOT_PRESENT; + file_octets[source_id] = STAT_NOT_PRESENT; partial_inspected_octets[source_id] = 0; section_size_target[source_id] = 0; stretch_section_to_packet[source_id] = false; @@ -265,6 +266,7 @@ void HttpFlowData::show(FILE* out_file) const fprintf(out_file, "File depth remaining: %" PRIi64 "/%" PRIi64 "\n", file_depth_remaining[0], file_depth_remaining[1]); fprintf(out_file, "Body octets: %" PRIi64 "/%" PRIi64 "\n", body_octets[0], body_octets[1]); + fprintf(out_file, "File octets: %" PRIi64 "/%" PRIi64 "\n", file_octets[0], file_octets[1]); fprintf(out_file, "Pipelining: front %d back %d overflow %d underflow %d\n", pipeline_front, pipeline_back, pipeline_overflow, pipeline_underflow); fprintf(out_file, "Cutter: %s/%s\n", (cutter[0] != nullptr) ? "Present" : "nullptr", diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index 43a47e6dd..b1a8ff3d4 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -161,6 +161,8 @@ private: // number of user data octets seen so far (regular body or chunks) int64_t body_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT }; + // normalized octets forwarded to file or MIME processing + int64_t file_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT }; uint32_t partial_inspected_octets[2] = { 0, 0 }; uint8_t* partial_detect_buffer[2] = { nullptr, nullptr }; uint32_t partial_detect_length[2] = { 0, 0 }; diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index 8082399fc..a5034ae53 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -279,7 +279,7 @@ void HttpMsgBody::do_file_processing(const Field& file_data) } if (file_flows->file_process(p, file_index, file_data.start(), fp_length, - body_octets + session_data->partial_inspected_octets[source_id], dir, + session_data->file_octets[source_id], dir, transaction->get_file_processing_id(source_id), file_position)) { session_data->file_depth_remaining[source_id] -= fp_length; @@ -303,11 +303,15 @@ void HttpMsgBody::do_file_processing(const Field& file_data) // file processing doesn't want any more data session_data->file_depth_remaining[source_id] = 0; } + session_data->file_octets[source_id] += fp_length; } else { + // FIXIT-M this interface does not convey any indication of end of message body. If the + // message body ends in the middle of a MIME message the partial file will not be flushed. session_data->mime_state[source_id]->process_mime_data(p, file_data.start(), file_data.length(), true, SNORT_FILE_POSITION_UNKNOWN); + session_data->file_octets[source_id] += file_data.length(); } } diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index 916116236..d0ce5063e 100644 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -401,6 +401,8 @@ void HttpMsgHeader::prepare_body() void HttpMsgHeader::setup_file_processing() { + session_data->file_octets[source_id] = 0; + const int64_t max_file_depth = FileService::get_max_file_depth(); if (max_file_depth <= 0) { diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index a5ca4eae9..e26451dbb 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -125,31 +125,44 @@ bool HttpStreamSplitter::finish(Flow* flow) if (!file_flows) return false; - const FileDirection dir = source_id == SRC_SERVER ? FILE_DOWNLOAD : FILE_UPLOAD; + const FileDirection dir = (source_id == SRC_SERVER) ? FILE_DOWNLOAD : FILE_UPLOAD; size_t file_index = 0; - uint64_t file_processing_id = 0; - // FIXIT-L How can there be a file in progress and no transaction in this direction? - if (session_data->transaction[source_id] != nullptr) + assert(session_data->transaction[source_id] != nullptr); + HttpMsgRequest* request = session_data->transaction[source_id]->get_request(); + if ((request != nullptr) and (request->get_http_uri() != nullptr)) { - HttpMsgRequest* request = session_data->transaction[source_id]->get_request(); - if ((request != nullptr) and (request->get_http_uri() != nullptr)) - { - file_index = request->get_http_uri()->get_file_proc_hash(); - } - file_processing_id = - session_data->transaction[source_id]->get_file_processing_id(source_id); + file_index = request->get_http_uri()->get_file_proc_hash(); } + const uint64_t file_processing_id = + session_data->transaction[source_id]->get_file_processing_id(source_id); file_flows->file_process(packet, file_index, nullptr, 0, 0, dir, file_processing_id, SNORT_FILE_END); +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), + "File processing finalization during finish()\n"); + fflush(HttpTestManager::get_output_file()); + } +#endif } else { + // FIXIT-M The following call does not actually accomplish anything. The MIME interface + // needs to be enhanced so that we can communicate end-of-data without side effects. session_data->mime_state[source_id]->process_mime_data(packet, nullptr, 0, true, SNORT_FILE_POSITION_UNKNOWN); delete session_data->mime_state[source_id]; session_data->mime_state[source_id] = nullptr; +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), "MIME finalization during finish()\n"); + fflush(HttpTestManager::get_output_file()); + } +#endif } return false; }