From: Vitalii Tron -X (vtron - SOFTSERVE INC at Cisco) Date: Thu, 1 May 2025 03:46:26 +0000 (+0000) Subject: Pull request #4699: http_inspect: add dynamic length-limited publishing of request... X-Git-Tag: 3.7.4.0~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9ae0675ccab85e1a771b5760ec18142d146f9016;p=thirdparty%2Fsnort3.git Pull request #4699: http_inspect: add dynamic length-limited publishing of request and response body Merge in SNORT/snort3 from ~VTRON/snort3:publish_http_body to master Squashed commit of the following: commit 2dba6d67d600da2f03621ce84dd10bda0486b926 Author: Vitalii Tron Date: Tue Oct 22 13:17:54 2024 -0400 http_inspect: add dynamic length-limited publishing of request and response body --- diff --git a/src/pub_sub/CMakeLists.txt b/src/pub_sub/CMakeLists.txt index 0782021d3..7c8e05c3d 100644 --- a/src/pub_sub/CMakeLists.txt +++ b/src/pub_sub/CMakeLists.txt @@ -18,6 +18,8 @@ set (PUB_SUB_INCLUDES http_event_ids.h http_events.h http_request_body_event.h + http_body_event.h + http_publish_length_event.h http_transaction_end_event.h intrinsic_event_ids.h netflow_event.h @@ -40,6 +42,7 @@ add_library( pub_sub OBJECT http_events.cc dns_events.cc http_request_body_event.cc + http_body_event.cc http_transaction_end_event.cc sip_events.cc ) diff --git a/src/pub_sub/http_body_event.cc b/src/pub_sub/http_body_event.cc new file mode 100644 index 000000000..9386d4fa8 --- /dev/null +++ b/src/pub_sub/http_body_event.cc @@ -0,0 +1,32 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http_body_event.cc author Vitalii Tron + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "http_body_event.h" + +using namespace snort; + +const uint8_t* HttpBodyEvent::get_body(int32_t& length) const +{ + length = http_body_length; + return http_body_ptr; +} diff --git a/src/pub_sub/http_body_event.h b/src/pub_sub/http_body_event.h new file mode 100644 index 000000000..f68f71bc9 --- /dev/null +++ b/src/pub_sub/http_body_event.h @@ -0,0 +1,49 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http_body_event.h author Vitalii Tron + +#ifndef HTTP_BODY_EVENT_H +#define HTTP_BODY_EVENT_H + +#include "framework/data_bus.h" + +namespace snort +{ + +// This event is published each time new request or response body data is received by http_inspect for HTTP traffic. +// The body may be published in several pieces, concluded by the publication with last_piece == true. +class SO_PUBLIC HttpBodyEvent : public snort::DataEvent +{ +public: + HttpBodyEvent(const uint8_t* http_body_ptr, const int32_t http_body_length, + const bool is_data_originates_from_client, const bool last_piece) + : http_body_ptr(http_body_ptr), http_body_length(http_body_length), + is_data_originates_from_client(is_data_originates_from_client), last_piece(last_piece) { } + const uint8_t* get_body(int32_t& length) const; + bool is_data_from_client() const { return is_data_originates_from_client; } + bool is_last_piece() const { return last_piece; } + +private: + const uint8_t* http_body_ptr; + const int32_t http_body_length; + const bool is_data_originates_from_client; + const bool last_piece; +}; + +} // namespace snort +#endif diff --git a/src/pub_sub/http_event_ids.h b/src/pub_sub/http_event_ids.h index 6b77a080e..febbb55e5 100644 --- a/src/pub_sub/http_event_ids.h +++ b/src/pub_sub/http_event_ids.h @@ -35,7 +35,9 @@ struct HttpEventIds REQUEST_HEADER, RESPONSE_HEADER, REQUEST_BODY, - END_OF_TRANSACTION, + BODY, + END_OF_TRANSACTION, + HTTP_PUBLISH_LENGTH, num_ids }; }; diff --git a/src/pub_sub/http_publish_length_event.h b/src/pub_sub/http_publish_length_event.h new file mode 100644 index 000000000..d681bd4bc --- /dev/null +++ b/src/pub_sub/http_publish_length_event.h @@ -0,0 +1,65 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http_publish_length_event.h author Steve Chew + +#ifndef HTTP_PUBLISH_LENGTH_EVENT_H +#define HTTP_PUBLISH_LENGTH_EVENT_H + +// An event to dynamically update the publish length used by http_inspect +// Subscribers MUST retain the given publish length if it's larger than +// the one they desire. That way all subscribers can work together to +// set the publish length. + +namespace snort +{ + +class SO_PUBLIC HttpPublishLengthEvent : public snort::DataEvent +{ +public: + HttpPublishLengthEvent(bool is_data_originates_from_client, int32_t publish_length) : + is_data_originates_from_client(is_data_originates_from_client), publish_length(publish_length) + { + } + + bool is_data_from_client() const + { return is_data_originates_from_client; } + + int32_t get_publish_length() + { return publish_length; } + + void set_publish_length(int32_t new_length) + { + if (new_length > 0) + { + publish_length = new_length; + publish_body = true; + } + } + + bool should_publish_body() const + { return publish_body; } + +private: + bool is_data_originates_from_client; + int32_t publish_length; + bool publish_body = false; +}; + +} + +#endif diff --git a/src/pub_sub/http_request_body_event.cc b/src/pub_sub/http_request_body_event.cc index 435e56ba1..dab3851db 100644 --- a/src/pub_sub/http_request_body_event.cc +++ b/src/pub_sub/http_request_body_event.cc @@ -34,13 +34,14 @@ const uint8_t* HttpRequestBodyEvent::get_request_body_data(int32_t& length, int3 { offset = msg_offset; - if (http_msg_body) + if (http_msg_body and (publish_length > 0)) { const Field& body = http_msg_body->get_msg_text_new(); - length = http_msg_body->get_publish_length(); - if (length > 0) + const auto body_length = body.length(); + + if (body_length > 0) { - assert(body.length() >= length); + length = publish_length > body_length ? body_length : publish_length; return body.start(); } } diff --git a/src/pub_sub/http_request_body_event.h b/src/pub_sub/http_request_body_event.h index 2f83eadd3..0edeefe4b 100644 --- a/src/pub_sub/http_request_body_event.h +++ b/src/pub_sub/http_request_body_event.h @@ -34,8 +34,10 @@ namespace snort class SO_PUBLIC HttpRequestBodyEvent : public snort::DataEvent { public: - HttpRequestBodyEvent(HttpMsgBody* msg_body, int32_t offset, bool last, HttpFlowData* flow_data) - : http_msg_body(msg_body), msg_offset(offset), last_piece(last), http_flow_data(flow_data) + HttpRequestBodyEvent(HttpMsgBody* msg_body, int32_t publish_length, int32_t offset, bool last, + HttpFlowData* flow_data) + : http_msg_body(msg_body), publish_length(publish_length), msg_offset(offset), last_piece(last), + http_flow_data(flow_data) { } const uint8_t* get_request_body_data(int32_t& length, int32_t& offset); @@ -46,10 +48,13 @@ public: private: HttpMsgBody* const http_msg_body; + // Length to be published, might be smaller than the body length due to REQUEST_PUBLISH_DEPTH + int32_t publish_length; const int32_t msg_offset; const bool last_piece; HttpFlowData* const http_flow_data; }; + } #endif diff --git a/src/pub_sub/test/CMakeLists.txt b/src/pub_sub/test/CMakeLists.txt index 00dddc52e..9bb74a5af 100644 --- a/src/pub_sub/test/CMakeLists.txt +++ b/src/pub_sub/test/CMakeLists.txt @@ -13,6 +13,10 @@ add_cpputest( pub_sub_http_request_body_event_test ../http_request_body_event.cc ../../service_inspectors/http_inspect/http_msg_body_cl.cc ) +add_cpputest( pub_sub_http_body_event_test + SOURCES + ../http_body_event.cc +) add_cpputest( pub_sub_eve_process_event_test SOURCES ../eve_process_event.h diff --git a/src/pub_sub/test/pub_sub_http_body_event_test.cc b/src/pub_sub/test/pub_sub_http_body_event_test.cc new file mode 100644 index 000000000..d190f33fc --- /dev/null +++ b/src/pub_sub/test/pub_sub_http_body_event_test.cc @@ -0,0 +1,81 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// pub_sub_http_body_event_test.cc author Vitalii Tron + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "pub_sub/http_body_event.h" + +#include +#include + +using namespace snort; + +TEST_GROUP(pub_sub_http_body_event_test) +{ + +}; + +TEST(pub_sub_http_body_event_test, regular_event) +{ + int32_t in_message_length = 500; + std::string in_message(in_message_length, 'A'); + const bool in_is_data_from_client = true; + const bool last_piece = true; + + HttpBodyEvent event((const uint8_t*)in_message.c_str(), in_message_length, in_is_data_from_client, last_piece); + + int32_t out_message_length; + const uint8_t* out_message = event.get_body(out_message_length); + + const bool out_is_data_from_client = event.is_data_from_client(); + const bool out_last_piece = event.is_last_piece(); + + CHECK(out_message_length == in_message_length); + CHECK(memcmp(out_message, in_message.c_str(), out_message_length) == 0); + CHECK(out_is_data_from_client == in_is_data_from_client); + CHECK(out_last_piece == last_piece); +} + +TEST(pub_sub_http_body_event_test, empty_data_event) +{ + const bool in_is_data_from_client = true; + const bool last_piece = false; + + HttpBodyEvent event(nullptr, 0, in_is_data_from_client, last_piece); + + int32_t out_message_length; + const uint8_t* out_message = event.get_body(out_message_length); + const bool out_is_data_from_client = event.is_data_from_client(); + const bool out_last_piece = event.is_last_piece(); + + CHECK(out_message == nullptr); + CHECK(out_message_length == 0); + CHECK(out_is_data_from_client == in_is_data_from_client); + CHECK(out_last_piece == last_piece); +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/pub_sub/test/pub_sub_http_request_body_event_test.cc b/src/pub_sub/test/pub_sub_http_request_body_event_test.cc index 35da286da..5c23539a7 100644 --- a/src/pub_sub/test/pub_sub_http_request_body_event_test.cc +++ b/src/pub_sub/test/pub_sub_http_request_body_event_test.cc @@ -129,6 +129,7 @@ TEST_GROUP(pub_sub_http_request_body_event_test) TEST(pub_sub_http_request_body_event_test, first_event) { int32_t msg_len = 500; + int32_t max_pub_len = 2000; int32_t length, offset; uint32_t stream_id = 1; std::string msg(msg_len, 'A'); @@ -136,7 +137,7 @@ TEST(pub_sub_http_request_body_event_test, first_event) mock().setData("stream_id", stream_id); HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr, HttpCommon::SRC_CLIENT, false, nullptr, nullptr); - HttpRequestBodyEvent event(body, 0, false, nullptr); + HttpRequestBodyEvent event(body, max_pub_len, 0, false, nullptr); const uint8_t* data = event.get_request_body_data(length, offset); CHECK(memcmp(data, msg.data(), length) == 0); CHECK(length == msg_len); @@ -149,6 +150,7 @@ TEST(pub_sub_http_request_body_event_test, first_event) TEST(pub_sub_http_request_body_event_test, last_event) { int32_t msg_len = 500; + int32_t max_pub_len = 2000; int32_t in_offset = REQUEST_PUBLISH_DEPTH - msg_len; int32_t length, offset; uint32_t stream_id = 3; @@ -157,7 +159,7 @@ TEST(pub_sub_http_request_body_event_test, last_event) mock().setData("pub_length", msg_len); HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr, HttpCommon::SRC_CLIENT, false, nullptr, nullptr); - HttpRequestBodyEvent event(body, in_offset, true, nullptr); + HttpRequestBodyEvent event(body, max_pub_len, in_offset, true, nullptr); const uint8_t* data = event.get_request_body_data(length, offset); CHECK(memcmp(data, msg.data(), length) == 0); CHECK(length == msg_len); @@ -170,10 +172,11 @@ TEST(pub_sub_http_request_body_event_test, last_event) TEST(pub_sub_http_request_body_event_test, empty_data_last_event) { int32_t in_offset = 1500; + int32_t max_pub_len = 2000; int32_t length, offset; uint32_t stream_id = 5; mock().setData("stream_id", stream_id); - HttpRequestBodyEvent event(nullptr, in_offset, true, nullptr); + HttpRequestBodyEvent event(nullptr, max_pub_len, in_offset, true, nullptr); const uint8_t* data = event.get_request_body_data(length, offset); CHECK(data == nullptr); CHECK(length == 0); @@ -182,6 +185,49 @@ TEST(pub_sub_http_request_body_event_test, empty_data_last_event) CHECK(event.is_last_request_body_piece()); } + +TEST(pub_sub_http_request_body_event_test, publish_length_lt_message_length) +{ + int32_t msg_len = 500; + int32_t max_pub_len = 200; + int32_t length, offset; + uint32_t stream_id = 1; + std::string msg(msg_len, 'A'); + mock().setData("pub_length", msg_len); + mock().setData("stream_id", stream_id); + HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr, + HttpCommon::SRC_CLIENT, false, nullptr, nullptr); + HttpRequestBodyEvent event(body, max_pub_len, 0, false, nullptr); + const uint8_t* data = event.get_request_body_data(length, offset); + CHECK(memcmp(data, msg.data(), length) == 0); + CHECK(length == max_pub_len); + CHECK(offset == 0); + CHECK(event.get_httpx_stream_id() == stream_id); + CHECK_FALSE(event.is_last_request_body_piece()); + delete body; +} + +TEST(pub_sub_http_request_body_event_test, zero_publish_length) +{ + int32_t msg_len = 500; + int32_t max_pub_len = 0; + int32_t length, offset; + uint32_t stream_id = 1; + std::string msg(msg_len, 'A'); + mock().setData("pub_length", msg_len); + mock().setData("stream_id", stream_id); + HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr, + HttpCommon::SRC_CLIENT, false, nullptr, nullptr); + HttpRequestBodyEvent event(body, max_pub_len, 0, false, nullptr); + const uint8_t* data = event.get_request_body_data(length, offset); + CHECK(memcmp(data, msg.data(), length) == 0); + CHECK(length == max_pub_len); + CHECK(offset == 0); + CHECK(event.get_httpx_stream_id() == stream_id); + CHECK_FALSE(event.is_last_request_body_piece()); + delete body; +} + int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/service_inspectors/http_inspect/http_common.h b/src/service_inspectors/http_inspect/http_common.h index 1917fa541..4dac4bd99 100644 --- a/src/service_inspectors/http_inspect/http_common.h +++ b/src/service_inspectors/http_inspect/http_common.h @@ -22,6 +22,9 @@ #include +#define STASH_PUBLISH_REQUEST_BODY "publish_request_body" +#define STASH_PUBLISH_RESPONSE_BODY "publish_response_body" + namespace HttpCommon { // Field status codes for when no valid value is present in length or integer value. Positive diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index ab621032e..6ad049d19 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -137,7 +137,7 @@ void HttpFlowData::half_reset(SourceId source_id) data_length[source_id] = STAT_NOT_PRESENT; body_octets[source_id] = STAT_NOT_PRESENT; file_octets[source_id] = STAT_NOT_PRESENT; - publish_octets[source_id] = STAT_NOT_PRESENT; + partial_inspected_octets[source_id] = 0; section_size_target[source_id] = 0; stretch_section_to_packet[source_id] = false; diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index cd9973ce1..4033673f1 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -30,6 +30,7 @@ #include "helpers/buffer_data.h" #include "js_norm/js_enum.h" #include "pub_sub/http_request_body_event.h" +#include "pub_sub/http_body_event.h" #include "http_api.h" #include "http_common.h" @@ -79,24 +80,66 @@ void HttpMsgBody::publish(unsigned pub_id) if (publish_length <= 0) return; + const bool is_request = (source_id == SRC_CLIENT); const int32_t& pub_depth_remaining = session_data->publish_depth_remaining[source_id]; int32_t& publish_octets = session_data->publish_octets[source_id]; const bool last_piece = (session_data->cutter[source_id] == nullptr) || tcp_close || (pub_depth_remaining == 0); - HttpRequestBodyEvent http_request_body_event(this, publish_octets, last_piece, session_data); + // Publish entire request/response body limited dynamically + int32_t should_publish_body = 0; + if (is_request) + flow->stash->get(STASH_PUBLISH_REQUEST_BODY, should_publish_body); + else + flow->stash->get(STASH_PUBLISH_RESPONSE_BODY, should_publish_body); - DataBus::publish(pub_id, HttpEventIds::REQUEST_BODY, http_request_body_event, flow); - publish_octets += publish_length; -#ifdef REG_TEST - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + if (should_publish_body) { - fprintf(HttpTestManager::get_output_file(), - "Published %" PRId32 " bytes of request body. last: %s\n", publish_length, - (last_piece ? "true" : "false")); - fflush(HttpTestManager::get_output_file()); + HttpBodyEvent http_body_event(msg_text_new.start(), publish_length, is_request, + last_piece); + DataBus::publish(pub_id, HttpEventIds::BODY, http_body_event, flow); + #ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), + "Published %" PRId32 " bytes of body. Originated from %s. last: %s\n", + publish_length, (is_request ? "client" : "server"), + (last_piece ? "true" : "false")); + fflush(HttpTestManager::get_output_file()); + } + #endif } -#endif + + // Publish request body limited statically + if (params->publish_request_body and is_request and (publish_octets < REQUEST_PUBLISH_DEPTH)) + { + // Exclude already published octets (publish_octets): + const auto request_publish_depth_remaining = REQUEST_PUBLISH_DEPTH - publish_octets; + + // We should not publish more than the remaining publish depth: + auto request_publish_length = (publish_length > request_publish_depth_remaining) ? + request_publish_depth_remaining : publish_length; + + // If it is not the last piece of the request, it should be marked as such because of REQUEST_PUBLISH_DEPTH limit: + // if sum of already published octets (publish_octets) and current publishing length (request_publish_length) + // is greater than the request publish depth. + auto request_last_piece = last_piece ? + true : (publish_octets + request_publish_length >= REQUEST_PUBLISH_DEPTH); + + HttpRequestBodyEvent http_request_body_event(this, request_publish_length, publish_octets, request_last_piece, session_data); + DataBus::publish(pub_id, HttpEventIds::REQUEST_BODY, http_request_body_event, flow); + #ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), + "Published %" PRId32 " bytes of request body. last: %s\n", request_publish_length, + (request_last_piece ? "true" : "false")); + fflush(HttpTestManager::get_output_file()); + } + #endif + } + + publish_octets += publish_length; } void HttpMsgBody::bookkeeping_regular_flush(uint32_t& partial_detect_length, @@ -123,6 +166,7 @@ void HttpMsgBody::clean_partial(uint32_t& partial_inspected_octets, uint32_t& pa const int32_t detect_length = (partial_js_detect_length <= session_data->detect_depth_remaining[source_id]) ? partial_js_detect_length : session_data->detect_depth_remaining[source_id]; + bookkeeping_regular_flush(partial_detect_length, partial_detect_buffer, partial_js_detect_length, detect_length); } @@ -194,11 +238,11 @@ void HttpMsgBody::analyze() } else mime_bufs = new std::list; - + while (ptr < section_end) { // After process_mime_data(), ptr will point to the last byte processed in the current MIME part - ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr, + ptr = session_data->mime_state[source_id]->process_mime_data(p, ptr, (section_end - ptr), true, SNORT_FILE_POSITION_UNKNOWN, &latest_attachment); ptr++; diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index c686f7aa3..ef3d3afcf 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -30,7 +30,9 @@ #include "file_api/file_service.h" #include "hash/hash_key_operations.h" #include "pub_sub/http_events.h" +#include "pub_sub/http_event_ids.h" #include "pub_sub/http_request_body_event.h" +#include "pub_sub/http_publish_length_event.h" #include "service_inspectors/http2_inspect/http2_flow_data.h" #include "sfip/sf_ip.h" @@ -483,11 +485,13 @@ void HttpMsgHeader::update_flow() // Common activities of preparing for upcoming body void HttpMsgHeader::prepare_body() { + const bool is_request = (source_id == SRC_CLIENT); + session_data->body_octets[source_id] = 0; setup_mime(); if (!session_data->mime_state[source_id]) { - const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth : + const int64_t& depth = is_request ? params->request_depth : params->response_depth; session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX; } @@ -499,11 +503,31 @@ void HttpMsgHeader::prepare_body() // body session_data->detect_depth_remaining[source_id] = INT64_MAX; } - if ((source_id == SRC_CLIENT) and params->publish_request_body) + + // Set the default depth if we're publishing the request body. + if (params->publish_request_body and is_request) { session_data->publish_octets[source_id] = 0; session_data->publish_depth_remaining[source_id] = REQUEST_PUBLISH_DEPTH; } + + // Dynamically update the publish depth. + int32_t cur_depth = session_data->publish_depth_remaining[source_id]; + HttpPublishLengthEvent http_publish_length_event = { is_request, cur_depth }; + DataBus::publish(DataBus::get_id(http_pub_key), HttpEventIds::HTTP_PUBLISH_LENGTH, http_publish_length_event, flow); + int32_t new_depth = http_publish_length_event.get_publish_length(); + + if (is_request) + flow->stash->store(STASH_PUBLISH_REQUEST_BODY, http_publish_length_event.should_publish_body()); + else + flow->stash->store(STASH_PUBLISH_RESPONSE_BODY, http_publish_length_event.should_publish_body()); + + if (new_depth > session_data->publish_depth_remaining[source_id]) + { + session_data->publish_octets[source_id] = 0; + session_data->publish_depth_remaining[source_id] = new_depth; + } + setup_file_processing(); setup_encoding_decompression(); setup_utf_decoding(); @@ -513,7 +537,7 @@ void HttpMsgHeader::prepare_body() if ((source_id == SRC_SERVER) && (params->script_detection)) session_data->accelerated_blocking[source_id] = true; - if (source_id == SRC_CLIENT) + if (is_request) { HttpModule::increment_peg_counts(PEG_REQUEST_BODY); diff --git a/src/service_inspectors/http_inspect/http_msg_section.cc b/src/service_inspectors/http_inspect/http_msg_section.cc index 68342a76e..903697716 100644 --- a/src/service_inspectors/http_inspect/http_msg_section.cc +++ b/src/service_inspectors/http_inspect/http_msg_section.cc @@ -21,12 +21,10 @@ #include "config.h" #endif -#include +#include "http_msg_section.h" #include "service_inspectors/http2_inspect/http2_flow_data.h" -#include "http_msg_section.h" - #include "http_context_data.h" #include "http_common.h" #include "http_enum.h" diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index c8bff64cd..70337abc9 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -25,6 +25,7 @@ #include "file_api/file_flows.h" #include "pub_sub/http_request_body_event.h" +#include "pub_sub/http_body_event.h" #include "http_common.h" #include "http_cutter.h" @@ -194,20 +195,45 @@ bool HttpStreamSplitter::finish(Flow* flow) #endif } } - // If we were publishing a request body need to publish that body is complete + + // If we were publishing a request or response body need to publish that body is complete if (session_data->publish_depth_remaining[source_id] > 0) { - HttpRequestBodyEvent http_request_body_event(nullptr, - session_data->publish_octets[source_id], true, session_data); - DataBus::publish(my_inspector->get_pub_id(), HttpEventIds::REQUEST_BODY, http_request_body_event, flow); -#ifdef REG_TEST - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + int32_t should_publish_body = 0; + if (source_id == SRC_CLIENT) + flow->stash->get(STASH_PUBLISH_REQUEST_BODY, should_publish_body); + else + flow->stash->get(STASH_PUBLISH_RESPONSE_BODY, should_publish_body); + + if (should_publish_body) { - fprintf(HttpTestManager::get_output_file(), - "Request body event published during finish()\n"); - fflush(HttpTestManager::get_output_file()); + HttpBodyEvent http_body_event(nullptr, 0, (source_id == SRC_CLIENT), true); + DataBus::publish(my_inspector->get_pub_id(), HttpEventIds::BODY, http_body_event, flow); +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), + "Http Body event published during finish(). Originated from %s.\n", + (source_id == HttpCommon::SourceId::SRC_CLIENT ? "client" : "server")); + fflush(HttpTestManager::get_output_file()); + } +#endif } + + if ((source_id == SRC_CLIENT) and (session_data->publish_octets[source_id] < REQUEST_PUBLISH_DEPTH)) + { + HttpRequestBodyEvent http_request_body_event(nullptr, 0, + session_data->publish_octets[source_id], true, session_data); + DataBus::publish(my_inspector->get_pub_id(), HttpEventIds::REQUEST_BODY, http_request_body_event, flow); +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) + { + fprintf(HttpTestManager::get_output_file(), + "Request body event published during finish()\n"); + fflush(HttpTestManager::get_output_file()); + } #endif + } } return false; }