appid_ha.h
appid_http_session.cc
appid_http_session.h
+ appid_http2_req_body_event_handler.h
appid_opportunistic_tls_event_handler.h
appid_peg_counts.h
appid_peg_counts.cc
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// appid_http2_req_body_event_handler.h
+// author Kani<kamurthi@cisco.com>
+
+#ifndef APPID_HTTP2_REQ_BODY_EVENT_HANDLER_H
+#define APPID_HTTP2_REQ_BODY_EVENT_HANDLER_H
+
+#include "pub_sub/http_request_body_event.h"
+
+class AppIdHttp2ReqBodyEventHandler : public snort::DataHandler
+{
+public:
+ AppIdHttp2ReqBodyEventHandler() : DataHandler(MOD_NAME){ }
+ void handle(snort::DataEvent& event, snort::Flow* flow) override
+ {
+ if (!pkt_thread_odp_ctxt)
+ return;
+ assert(flow);
+ snort::Packet* p = snort::DetectionEngine::get_current_packet();
+ assert(p);
+ AppIdSession* asd = snort::appid_api.get_appid_session(*flow);
+
+ if (!asd or
+ !asd->get_session_flags(APPID_SESSION_DISCOVER_APP | APPID_SESSION_SPECIAL_MONITORED))
+ return;
+ // Skip sessions using old odp context after reload detectors
+ if (pkt_thread_odp_ctxt->get_version() != asd->get_odp_ctxt_version())
+ return;
+ snort::HttpRequestBodyEvent* http_req_body = (snort::HttpRequestBodyEvent*)&event;
+ AppIdHttpSession* hsession = asd->get_matching_http_session(
+ http_req_body->get_http2_stream_id());
+
+ if (!hsession)
+ return;
+
+ const uint8_t* header_start;
+ int32_t header_length;
+ int32_t offset;
+ AppidChangeBits change_bits;
+ header_start = http_req_body->get_request_body_data(header_length, offset);
+ if (hsession->get_field(REQ_BODY_FID) and
+ !asd->get_session_flags(APPID_SESSION_APP_REINSPECT))
+ hsession->set_chp_finished(false);
+
+ hsession->set_req_body_field(REQ_BODY_FID, header_start, header_length, change_bits);
+ hsession->process_http_packet(APP_ID_FROM_INITIATOR, change_bits,
+ asd->get_odp_ctxt().get_http_matchers());
+ asd->publish_appid_event(change_bits, *p, true, asd->get_api().get_hsessions_size() - 1);
+
+ bool last_req_rcvd = http_req_body->is_last_request_body_piece();
+ if (last_req_rcvd)
+ hsession->set_rcvd_full_req_body(last_req_rcvd);
+ }
+};
+#endif
+
}
}
+void AppIdHttpSession::set_req_body_field(HttpFieldIds id, const uint8_t* str, int32_t len,
+ AppidChangeBits& change_bits)
+{
+ if (str and len)
+ {
+ if (rcvd_full_req_body)
+ {
+ delete meta_data[id];
+ meta_data[id] = nullptr;
+ rcvd_full_req_body = false;
+ }
+
+ if (!meta_data[id])
+ meta_data[id] = new std::string((const char*)str, len);
+ else
+ {
+ std::string *req_body = new std::string(*meta_data[id]);
+ delete meta_data[id];
+ req_body->append((const char*)str);
+ meta_data[id] = req_body;
+ }
+ set_http_change_bits(change_bits, id);
+ set_scan_flags(id);
+
+ if (appidDebug->is_active())
+ print_field(id, meta_data[id]);
+ }
+}
void AppIdHttpSession::print_field(HttpFieldIds id, const std::string* field)
{
string field_name;
void update_url(AppidChangeBits& change_bits);
void set_field(HttpFieldIds id, const std::string* str, AppidChangeBits& change_bits);
void set_field(HttpFieldIds id, const uint8_t* str, int32_t len, AppidChangeBits& change_bits);
+ void set_req_body_field(HttpFieldIds id, const uint8_t* str, int32_t len,
+ AppidChangeBits& change_bits);
const std::string* get_field(HttpFieldIds id) const
{ return meta_data[id]; }
{
return http2_stream_id;
}
+ void set_rcvd_full_req_body(bool req_full_body)
+ {
+ rcvd_full_req_body = req_full_body;
+ }
+ bool get_rcvd_full_req_body()
+ {
+ return rcvd_full_req_body;
+ }
protected:
#endif
uint32_t http2_stream_id = 0;
bool is_payload_processed = false;
+ bool rcvd_full_req_body = false;
};
#endif
#include "appid_discovery.h"
#include "appid_ha.h"
#include "appid_http_event_handler.h"
+#include "appid_http2_req_body_event_handler.h"
#include "appid_opportunistic_tls_event_handler.h"
#include "appid_peg_counts.h"
#include "appid_session.h"
DataBus::subscribe_global(HTTP_RESPONSE_HEADER_EVENT_KEY, new HttpEventHandler(
HttpEventHandler::RESPONSE_EVENT, *this), sc);
+ DataBus::subscribe_global(HTTP2_REQUEST_BODY_EVENT_KEY, new AppIdHttp2ReqBodyEventHandler(),
+ sc);
+
DataBus::subscribe_global(DATA_DECRYPT_EVENT, new DataDecryptEventHandler(), sc);
DataBus::subscribe_global(DCERPC_EXP_SESSION_EVENT_KEY, new DceExpSsnEventHandler(), sc);
expect_events.h
finalize_packet_event.h
http_events.h
+ http_request_body_event.h
opportunistic_tls_event.h
sip_events.h
smb_events.h
${PUB_SUB_INCLUDES}
cip_events.cc
http_events.cc
+ http_request_body_event.cc
sip_events.cc
)
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_request_body_event.cc author Katura Harvey <katharve@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "http_request_body_event.h"
+
+#include "service_inspectors/http_inspect/http_flow_data.h"
+
+using namespace snort;
+
+const uint8_t* HttpRequestBodyEvent::get_request_body_data(int32_t& length, int32_t& offset)
+{
+ offset = msg_offset;
+
+ if (http_msg_body)
+ {
+ const Field& body = http_msg_body->get_msg_text_new();
+ length = http_msg_body->get_publish_length();
+ if (length > 0)
+ {
+ assert(body.length() >= length);
+ return body.start();
+ }
+ }
+
+ length = 0;
+ return nullptr;
+}
+
+bool HttpRequestBodyEvent::is_last_request_body_piece()
+{
+ return last_piece;
+}
+
+uint32_t HttpRequestBodyEvent::get_http2_stream_id() const
+{
+ return http_flow_data->get_h2_stream_id();
+}
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_request_body_event.h author Katura Harvey <katharve@cisco.com>
+
+#ifndef HTTP_REQUEST_BODY_EVENT_H
+#define HTTP_REQUEST_BODY_EVENT_H
+
+#include "framework/data_bus.h"
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_field.h"
+#include "service_inspectors/http_inspect/http_msg_body.h"
+
+// These are common values between the HTTP inspector and the subscribers.
+#define HTTP2_REQUEST_BODY_EVENT_KEY "http2_request_body_event"
+
+class HttpFlowData;
+
+namespace snort
+{
+// This event is published each time new request body data is received by http_inspect for HTTP/2
+// traffic, up to the publish depth. The full request body may be sent in several pieces
+class SO_PUBLIC HttpRequestBodyEvent : public snort::DataEvent
+{
+public:
+ HttpRequestBodyEvent(HttpMsgBody* msg_body, int32_t offset, bool last, HttpFlowData* flow_data)
+ : http_msg_body(msg_body), msg_offset(offset), last_piece(last), http_flow_data(flow_data)
+ { }
+
+ const uint8_t* get_request_body_data(int32_t& length, int32_t& offset);
+ bool is_last_request_body_piece();
+ uint32_t get_http2_stream_id() const;
+
+private:
+ const HttpMsgBody* const http_msg_body;
+ const int32_t msg_offset;
+ const bool last_piece;
+ HttpFlowData* const http_flow_data;
+};
+}
+#endif
+
SOURCES
../http_events.cc
)
+add_cpputest( pub_sub_http_request_body_event_test
+ SOURCES
+ ../http_request_body_event.cc
+ ../../service_inspectors/http_inspect/http_msg_body_cl.cc
+)
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// pub_sub_http_request_body_event_test.cc author Katura Harvey <katharve@cisco.com>
+
+// Unit test for the HttpRequestBodyEvent methods for HTTP/2 request bodies
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <cstring>
+
+#include "pub_sub/http_request_body_event.h"
+#include "service_inspectors/http_inspect/http_common.h"
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_field.h"
+#include "service_inspectors/http_inspect/http_msg_body_cl.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+#include <CppUTestExt/MockSupport.h>
+
+using namespace snort;
+using namespace HttpCommon;
+using namespace HttpEnums;
+
+// Stubs to make the code link
+HttpMsgBody::HttpMsgBody(const uint8_t* buffer, const uint16_t buf_size,
+ HttpFlowData* session_data_, SourceId source_id_, bool buf_owner, Flow* flow_,
+ const HttpParaList* params_):
+ HttpMsgSection(buffer, buf_size, session_data_, source_id_, buf_owner, flow_, params_),
+ body_octets(0),
+ first_body(0)
+{
+ msg_text_new.set(buf_size, buffer, buf_owner);
+ publish_length = buf_size;
+}
+void HttpMsgBody::analyze() {}
+void HttpMsgBody::publish() {}
+void HttpMsgBody::do_file_processing(const Field&) {}
+void HttpMsgBody::do_utf_decoding(const Field&, Field&) {}
+void HttpMsgBody::do_file_decompression(const Field&, Field&) {}
+void HttpMsgBody::do_js_normalization(const Field&, Field&, bool) {}
+void HttpMsgBody::clean_partial(uint32_t&, uint32_t&, uint8_t*&, uint32_t&, int32_t) {}
+void HttpMsgBody::bookkeeping_regular_flush(uint32_t&, uint8_t*&, uint32_t&, int32_t) {}
+#ifdef REG_TEST
+void HttpMsgBody::print_body_section(FILE*, const char*) {}
+#endif
+
+HttpMsgSection::HttpMsgSection(const uint8_t* buffer, const uint16_t buf_size,
+ HttpFlowData* session_data_, HttpCommon::SourceId source_id_, bool buf_owner,
+ snort::Flow* flow_, const HttpParaList* params_):
+ msg_text(buf_size, buffer, buf_owner),
+ session_data(session_data_),
+ flow(flow_),
+ params(params_),
+ transaction(HttpTransaction::attach_my_transaction(session_data, source_id_)),
+ trans_num(STAT_NOT_PRESENT),
+ status_code_num(STAT_NOT_PRESENT),
+ source_id(source_id_),
+ version_id(VERS__NO_SOURCE),
+ method_id(METH__NOT_PRESENT),
+ tcp_close(false)
+{}
+void HttpMsgSection::update_depth() const{}
+
+HttpTransaction*HttpTransaction::attach_my_transaction(HttpFlowData*, HttpCommon::SourceId)
+ { return nullptr; }
+Field::Field(int32_t length, const uint8_t* start, bool own_the_buffer_) :
+ strt(start), len(length), own_the_buffer(own_the_buffer_)
+{}
+void Field::set(int32_t length, const uint8_t* start, bool own_the_buffer_)
+{
+ assert(len == STAT_NOT_COMPUTE);
+ assert(strt == nullptr);
+ assert(start != nullptr);
+ assert(length >= 0);
+ assert(length <= MAX_OCTETS);
+ strt = start;
+ len = length;
+ own_the_buffer = own_the_buffer_;
+}
+
+void HttpFlowData::half_reset(HttpCommon::SourceId) {}
+
+int32_t HttpMsgBody::get_publish_length() const
+{
+ return mock().getData("pub_length").getIntValue();
+}
+
+uint32_t HttpFlowData::get_h2_stream_id() const
+{
+ return mock().getData("stream_id").getUnsignedIntValue();
+}
+
+
+TEST_GROUP(pub_sub_http_request_body_event_test)
+{
+ void teardown() override
+ {
+ mock().clear();
+ }
+};
+
+TEST(pub_sub_http_request_body_event_test, first_event)
+{
+ int32_t msg_len = 500;
+ int32_t length, offset;
+ uint32_t stream_id = 1;
+ std::string msg(msg_len, 'A');
+ mock().setData("pub_length", msg_len);
+ mock().setData("stream_id", stream_id);
+ HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr,
+ HttpCommon::SRC_CLIENT, false, nullptr, nullptr);
+ HttpRequestBodyEvent event(body, 0, false, nullptr);
+ const uint8_t* data = event.get_request_body_data(length, offset);
+ CHECK(memcmp(data, msg.data(), length) == 0);
+ CHECK(length == msg_len);
+ CHECK(offset == 0);
+ CHECK(event.get_http2_stream_id() == stream_id);
+ CHECK_FALSE(event.is_last_request_body_piece());
+ delete body;
+}
+
+TEST(pub_sub_http_request_body_event_test, last_event)
+{
+ int32_t msg_len = 500;
+ int32_t in_offset = REQUEST_PUBLISH_DEPTH - msg_len;
+ int32_t length, offset;
+ uint32_t stream_id = 3;
+ mock().setData("stream_id", stream_id);
+ std::string msg(msg_len, 'A');
+ mock().setData("pub_length", msg_len);
+ HttpMsgBody* body = new HttpMsgBodyCl((const uint8_t*)msg.c_str(), msg_len, nullptr,
+ HttpCommon::SRC_CLIENT, false, nullptr, nullptr);
+ HttpRequestBodyEvent event(body, in_offset, true, nullptr);
+ const uint8_t* data = event.get_request_body_data(length, offset);
+ CHECK(memcmp(data, msg.data(), length) == 0);
+ CHECK(length == msg_len);
+ CHECK(offset == 1500);
+ CHECK(event.get_http2_stream_id() == stream_id);
+ CHECK(event.is_last_request_body_piece());
+ delete body;
+}
+
+TEST(pub_sub_http_request_body_event_test, empty_data_last_event)
+{
+ int32_t in_offset = 1500;
+ int32_t length, offset;
+ uint32_t stream_id = 5;
+ mock().setData("stream_id", stream_id);
+ HttpRequestBodyEvent event(nullptr, in_offset, true, nullptr);
+ const uint8_t* data = event.get_request_body_data(length, offset);
+ CHECK(data == nullptr);
+ CHECK(length == 0);
+ CHECK(offset == 1500);
+ CHECK(event.get_http2_stream_id() == stream_id);
+ CHECK(event.is_last_request_body_piece());
+}
+
+int main(int argc, char** argv)
+{
+ return CommandLineTestRunner::RunAllTests(argc, argv);
+}
+
static const int MAX_OCTETS = 63780;
static const int GZIP_BLOCK_SIZE = 2048;
static const int MAX_SECTION_STRETCH = 1460;
+static const int REQUEST_PUBLISH_DEPTH = 2000;
static const uint32_t HTTP_GID = 119;
static const int GZIP_WINDOW_BITS = 31;
#include "http_flow_data.h"
#include "decompress/file_decomp.h"
+#include "service_inspectors/http2_inspect/http2_flow_data.h"
#include "utils/js_normalizer.h"
#include "http_cutter.h"
uint64_t HttpFlowData::instance_count = 0;
#endif
-HttpFlowData::HttpFlowData() : FlowData(inspector_id)
+HttpFlowData::HttpFlowData(Flow* flow) : FlowData(inspector_id)
{
#ifdef REG_TEST
if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
if (HttpModule::get_peg_counts(PEG_MAX_CONCURRENT_SESSIONS) <
HttpModule::get_peg_counts(PEG_CONCURRENT_SESSIONS))
HttpModule::increment_peg_counts(PEG_MAX_CONCURRENT_SESSIONS);
+
+ Http2FlowData* h2i_flow_data = nullptr;
+ if (Http2FlowData::inspector_id != 0)
+ h2i_flow_data = (Http2FlowData*)flow->get_flow_data(Http2FlowData::inspector_id);
+ if (h2i_flow_data != nullptr)
+ {
+ for_http2 = true;
+ h2_stream_id = h2i_flow_data->get_processing_stream_id();
+ }
}
HttpFlowData::~HttpFlowData()
data_length[source_id] = STAT_NOT_PRESENT;
body_octets[source_id] = STAT_NOT_PRESENT;
file_octets[source_id] = STAT_NOT_PRESENT;
+ publish_octets[source_id] = STAT_NOT_PRESENT;
partial_inspected_octets[source_id] = 0;
section_size_target[source_id] = 0;
stretch_section_to_packet[source_id] = false;
accelerated_blocking[source_id] = false;
file_depth_remaining[source_id] = STAT_NOT_PRESENT;
detect_depth_remaining[source_id] = STAT_NOT_PRESENT;
+ publish_depth_remaining[source_id] = STAT_NOT_PRESENT;
detection_status[source_id] = DET_REACTIVATING;
compression[source_id] = CMP_NONE;
}
}
+uint32_t HttpFlowData::get_h2_stream_id() const
+{
+ return h2_stream_id;
+}
+
#ifdef REG_TEST
void HttpFlowData::show(FILE* out_file) const
{
class HttpFlowData : public snort::FlowData
{
public:
- HttpFlowData();
+ HttpFlowData(snort::Flow* flow);
~HttpFlowData() override;
static unsigned inspector_id;
static void init() { inspector_id = snort::FlowData::create_flow_data_id(); }
void reset_partial_flush(HttpCommon::SourceId source_id)
{ partial_flush[source_id] = false; }
+ uint32_t get_h2_stream_id() const;
+
private:
// HTTP/2 handling
bool for_http2 = false;
HttpEnums::H2BodyState h2_body_state[2] = { HttpEnums::H2_BODY_NOT_COMPLETE,
HttpEnums::H2_BODY_NOT_COMPLETE };
+ uint32_t h2_stream_id = 0;
// Convenience routines
void half_reset(HttpCommon::SourceId source_id);
HttpCommon::STAT_NOT_PRESENT };
int64_t detect_depth_remaining[2] = { HttpCommon::STAT_NOT_PRESENT,
HttpCommon::STAT_NOT_PRESENT };
+ int32_t publish_depth_remaining[2] = { HttpCommon::STAT_NOT_PRESENT,
+ HttpCommon::STAT_NOT_PRESENT };
uint64_t expected_trans_num[2] = { 1, 1 };
// number of user data octets seen so far (regular body or chunks)
int64_t body_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
// normalized octets forwarded to file or MIME processing
int64_t file_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
+ int32_t publish_octets[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
uint32_t partial_inspected_octets[2] = { 0, 0 };
uint8_t* partial_detect_buffer[2] = { nullptr, nullptr };
uint32_t partial_detect_length[2] = { 0, 0 };
ConfigLogger::log_flag("plus_to_space", params->uri_param.plus_to_space);
ConfigLogger::log_flag("simplify_path", params->uri_param.simplify_path);
ConfigLogger::log_value("xff_headers", xff_headers.c_str());
+ ConfigLogger::log_flag("request_body_app_detection", params->publish_request_body);
}
InspectSection HttpInspect::get_latest_is(const Packet* p)
void HttpInspect::http_set_flow_data(Flow* flow, HttpFlowData* flow_data)
{
- Http2FlowData* h2i_flow_data = nullptr;
- if (Http2FlowData::inspector_id != 0)
- h2i_flow_data = (Http2FlowData*)flow->get_flow_data(Http2FlowData::inspector_id);
- if (h2i_flow_data == nullptr)
+ // for_http2 set in HttpFlowData constructor after checking for h2i_flow_data
+ if (!flow_data->for_http2)
flow->set_flow_data(flow_data);
else
{
- flow_data->for_http2 = true;
+ Http2FlowData* h2i_flow_data =
+ (Http2FlowData*)flow->get_flow_data(Http2FlowData::inspector_id);
+ assert(h2i_flow_data);
h2i_flow_data->set_hi_flow_data(flow_data);
}
}
{ "xff_headers", Parameter::PT_STRING, nullptr, "x-forwarded-for true-client-ip",
"specifies the xff type headers to parse and consider in the same order "
"of preference as defined" },
+
+ { "request_body_app_detection", Parameter::PT_BOOL, nullptr, "false",
+ "make HTTP/2 request message bodies available for application detection "
+ "(detection requires AppId)" },
+
#ifdef REG_TEST
{ "test_input", Parameter::PT_BOOL, nullptr, "false",
"read HTTP messages from text file" },
}
params->xff_headers[hdr_idx] = end_header;
}
+ else if (val.is("request_body_app_detection"))
+ {
+ params->publish_request_body = val.get_bool();
+ }
+
#ifdef REG_TEST
else if (val.is("test_input"))
{
bool decompress_zip = false;
bool script_detection = false;
snort::LiteralSearch::Handle* script_detection_handle = nullptr;
+ bool publish_request_body = false;
struct JsNormParam
{
#include "http_msg_body.h"
#include "file_api/file_flows.h"
+#include "pub_sub/http_request_body_event.h"
#include "http_api.h"
#include "http_common.h"
#include "http_js_norm.h"
#include "http_msg_header.h"
#include "http_msg_request.h"
+#include "http_test_manager.h"
using namespace snort;
using namespace HttpCommon;
get_related_sections();
}
+void HttpMsgBody::publish()
+{
+ if (publish_length <= 0)
+ return;
+
+ const int32_t& pub_depth_remaining = session_data->publish_depth_remaining[source_id];
+ int32_t& publish_octets = session_data->publish_octets[source_id];
+ const bool last_piece = (session_data->cutter[source_id] == nullptr) || tcp_close ||
+ (pub_depth_remaining == 0);
+
+ HttpRequestBodyEvent http_request_body_event(this, publish_octets, last_piece, session_data);
+
+ DataBus::publish(HTTP2_REQUEST_BODY_EVENT_KEY, http_request_body_event, flow);
+ publish_octets += publish_length;
+#ifdef REG_TEST
+ if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+ {
+ fprintf(HttpTestManager::get_output_file(),
+ "Published %" PRId32 " bytes of request body. last: %s\n", publish_length,
+ (last_piece ? "true" : "false"));
+ fflush(HttpTestManager::get_output_file());
+ }
+#endif
+}
+
void HttpMsgBody::bookkeeping_regular_flush(uint32_t& partial_detect_length,
uint8_t*& partial_detect_buffer, uint32_t& partial_js_detect_length, int32_t detect_length)
{
delete[] partial_detect_buffer;
session_data->update_deallocations(partial_detect_length);
assert(detect_length <= session_data->detect_depth_remaining[source_id]);
- bookkeeping_regular_flush(partial_detect_length, partial_detect_buffer, partial_js_detect_length,
- detect_length);
+ bookkeeping_regular_flush(partial_detect_length, partial_detect_buffer,
+ partial_js_detect_length, detect_length);
}
}
else
msg_text_new.set(msg_text);
- do_utf_decoding(msg_text_new, decoded_body);
-
- if (session_data->file_depth_remaining[source_id] > 0)
+ int32_t& pub_depth_remaining = session_data->publish_depth_remaining[source_id];
+ if (pub_depth_remaining > 0)
{
- do_file_processing(decoded_body);
+ publish_length = (pub_depth_remaining > msg_text_new.length()) ?
+ msg_text_new.length() : pub_depth_remaining;
+ pub_depth_remaining -= publish_length;
}
- if (session_data->detect_depth_remaining[source_id] > 0)
+ if (session_data->file_depth_remaining[source_id] > 0 or
+ session_data->detect_depth_remaining[source_id] > 0)
{
- do_file_decompression(decoded_body, decompressed_file_body);
+ do_utf_decoding(msg_text_new, decoded_body);
- uint32_t& partial_detect_length = session_data->partial_detect_length[source_id];
- uint8_t*& partial_detect_buffer = session_data->partial_detect_buffer[source_id];
- uint32_t& partial_js_detect_length = session_data->partial_js_detect_length[source_id];
+ if (session_data->file_depth_remaining[source_id] > 0)
+ {
+ do_file_processing(decoded_body);
+ }
- if (partial_detect_length > 0)
+ if (session_data->detect_depth_remaining[source_id] > 0)
{
- const int32_t total_length = partial_detect_length + decompressed_file_body.length();
- uint8_t* const cumulative_buffer = new uint8_t[total_length];
- memcpy(cumulative_buffer, partial_detect_buffer, partial_detect_length);
- memcpy(cumulative_buffer + partial_detect_length, decompressed_file_body.start(),
- decompressed_file_body.length());
- cumulative_data.set(total_length, cumulative_buffer, true);
+ do_file_decompression(decoded_body, decompressed_file_body);
- do_js_normalization(cumulative_data, js_norm_body, true);
+ uint32_t& partial_detect_length = session_data->partial_detect_length[source_id];
+ uint8_t*& partial_detect_buffer = session_data->partial_detect_buffer[source_id];
+ uint32_t& partial_js_detect_length = session_data->partial_js_detect_length[source_id];
- if ((int32_t)partial_js_detect_length == js_norm_body.length())
+ if (partial_detect_length > 0)
{
- clean_partial(partial_inspected_octets, partial_detect_length,
- partial_detect_buffer, partial_js_detect_length, js_norm_body.length());
- return;
+ const int32_t total_length = partial_detect_length + decompressed_file_body.length();
+ uint8_t* const cumulative_buffer = new uint8_t[total_length];
+ memcpy(cumulative_buffer, partial_detect_buffer, partial_detect_length);
+ memcpy(cumulative_buffer + partial_detect_length, decompressed_file_body.start(),
+ decompressed_file_body.length());
+ cumulative_data.set(total_length, cumulative_buffer, true);
+ do_js_normalization(cumulative_data, js_norm_body, true);
+ if ((int32_t)partial_js_detect_length == js_norm_body.length())
+ {
+ clean_partial(partial_inspected_octets, partial_detect_length,
+ partial_detect_buffer, partial_js_detect_length, js_norm_body.length());
+ return;
+ }
}
- }
- else
- do_js_normalization(decompressed_file_body, js_norm_body, false);
+ else
+ do_js_normalization(decompressed_file_body, js_norm_body, false);
- const int32_t detect_length =
- (js_norm_body.length() <= session_data->detect_depth_remaining[source_id]) ?
- js_norm_body.length() : session_data->detect_depth_remaining[source_id];
+ const int32_t detect_length =
+ (js_norm_body.length() <= session_data->detect_depth_remaining[source_id]) ?
+ js_norm_body.length() : session_data->detect_depth_remaining[source_id];
- detect_data.set(detect_length, js_norm_body.start());
+ detect_data.set(detect_length, js_norm_body.start());
- delete[] partial_detect_buffer;
- session_data->update_deallocations(partial_detect_length);
+ delete[] partial_detect_buffer;
+ session_data->update_deallocations(partial_detect_length);
- if (!session_data->partial_flush[source_id])
- {
- bookkeeping_regular_flush(partial_detect_length, partial_detect_buffer,
- partial_js_detect_length, detect_length);
- }
- else
- {
- Field* decompressed = (cumulative_data.length() > 0) ?
- &cumulative_data : &decompressed_file_body;
- uint8_t* const save_partial = new uint8_t[decompressed->length()];
- memcpy(save_partial, decompressed->start(), decompressed->length());
- partial_detect_buffer = save_partial;
- partial_detect_length = decompressed->length();
- partial_js_detect_length = js_norm_body.length();
- session_data->update_allocations(partial_detect_length);
- }
+ if (!session_data->partial_flush[source_id])
+ {
+ bookkeeping_regular_flush(partial_detect_length, partial_detect_buffer,
+ partial_js_detect_length, detect_length);
+ }
+ else
+ {
+ Field* decompressed = (cumulative_data.length() > 0) ?
+ &cumulative_data : &decompressed_file_body;
+ uint8_t* const save_partial = new uint8_t[decompressed->length()];
+ memcpy(save_partial, decompressed->start(), decompressed->length());
+ partial_detect_buffer = save_partial;
+ partial_detect_length = decompressed->length();
+ partial_js_detect_length = js_norm_body.length();
+ session_data->update_allocations(partial_detect_length);
+ }
- set_file_data(const_cast<uint8_t*>(detect_data.start()),
- (unsigned)detect_data.length());
+ set_file_data(const_cast<uint8_t*>(detect_data.start()),
+ (unsigned)detect_data.length());
+ }
}
-
body_octets += msg_text.length();
partial_inspected_octets = session_data->partial_flush[source_id] ? msg_text.length() : 0;
}
return classic_normalize(detect_data, classic_client_body, false, params->uri_param);
}
+int32_t HttpMsgBody::get_publish_length() const
+{
+ return publish_length;
+}
+
#ifdef REG_TEST
// Common elements of print_section() for body sections
void HttpMsgBody::print_body_section(FILE* output, const char* body_type_str)
HttpMsgBody* get_body() override { return this; }
const Field& get_classic_client_body();
const Field& get_detect_data() { return detect_data; }
+ const Field& get_msg_text_new() const { return msg_text_new; }
static void fd_event_callback(void* context, int event);
bool is_first() { return first_body; }
+ void publish() override;
+ int32_t get_publish_length() const;
protected:
HttpMsgBody(const uint8_t* buffer, const uint16_t buf_size, HttpFlowData* session_data_,
Field detect_data;
Field enhanced_js_norm_body;
Field classic_client_body; // URI normalization applied
+
+ int32_t publish_length = HttpCommon::STAT_NOT_PRESENT;
};
#endif
#include "file_api/file_service.h"
#include "hash/hash_key_operations.h"
#include "pub_sub/http_events.h"
+#include "pub_sub/http_request_body_event.h"
#include "service_inspectors/http2_inspect/http2_flow_data.h"
#include "sfip/sf_ip.h"
void HttpMsgHeader::publish()
{
- const uint32_t stream_id = get_h2_stream_id();
+ const uint32_t stream_id = session_data->get_h2_stream_id();
- HttpEvent http_event(this, session_data->for_http2, stream_id);
+ HttpEvent http_header_event(this, session_data->for_http2, stream_id);
const char* key = (source_id == SRC_CLIENT) ?
HTTP_REQUEST_HEADER_EVENT_KEY : HTTP_RESPONSE_HEADER_EVENT_KEY;
- DataBus::publish(key, http_event, flow);
+ DataBus::publish(key, http_header_event, flow);
}
const Field& HttpMsgHeader::get_true_ip()
const int64_t& depth = (source_id == SRC_CLIENT) ? params->request_depth :
params->response_depth;
session_data->detect_depth_remaining[source_id] = (depth != -1) ? depth : INT64_MAX;
+ if ((source_id == SRC_CLIENT) and params->publish_request_body and session_data->for_http2)
+ {
+ session_data->publish_octets[source_id] = 0;
+ session_data->publish_depth_remaining[source_id] = REQUEST_PUBLISH_DEPTH;
+ }
setup_file_processing();
setup_encoding_decompression();
setup_utf_decoding();
}
// Generate the unique file id for multi file processing
- set_multi_file_processing_id(get_transaction_id(), get_h2_stream_id());
+ set_multi_file_processing_id(get_transaction_id(), session_data->get_h2_stream_id());
// Do we meet all the conditions for MIME file processing?
if (source_id == SRC_CLIENT)
{
const int64_t& file_depth_remaining = session_data->file_depth_remaining[source_id];
const int64_t& detect_depth_remaining = session_data->detect_depth_remaining[source_id];
+ const int32_t& publish_depth_remaining = session_data->publish_depth_remaining[source_id];
if ((detect_depth_remaining <= 0) &&
(session_data->detection_status[source_id] == DET_ON) &&
if (detect_depth_remaining <= 0)
{
- if (file_depth_remaining <= 0)
+ if ((file_depth_remaining <= 0) && (publish_depth_remaining <= 0))
{
// Don't need any more of the body
session_data->section_size_target[source_id] = 0;
}
else
{
- // Just for file processing.
- session_data->section_size_target[source_id] = target_size;
+ // Need data for file processing or publishing
session_data->stretch_section_to_packet[source_id] = true;
+ session_data->section_size_target[source_id] = target_size;
}
return;
}
trailer[SRC_SERVER] = transaction->get_trailer(SRC_SERVER);
}
-uint32_t HttpMsgSection::get_h2_stream_id()
-{
- if (h2_stream_id != STAT_NOT_COMPUTE)
- return h2_stream_id;
-
- h2_stream_id = 0;
- if (session_data->for_http2)
- {
- Http2FlowData* h2i_flow_data =
- (Http2FlowData*)flow->get_flow_data(Http2FlowData::inspector_id);
- assert(h2i_flow_data);
- if (h2i_flow_data)
- h2_stream_id = h2i_flow_data->get_processing_stream_id();
- }
- return h2_stream_id;
-}
-
void HttpMsgSection::clear()
{
transaction->clear_section();
HttpEnums::MethodId method_id;
const bool tcp_close;
- int64_t h2_stream_id = HttpCommon::STAT_NOT_COMPUTE;
- uint32_t get_h2_stream_id();
-
// Pointers to related message sections in the same transaction
HttpMsgRequest* request;
HttpMsgStatus* status;
#endif
#include "file_api/file_flows.h"
+#include "pub_sub/http_request_body_event.h"
#include "http_common.h"
#include "http_cutter.h"
#include "http_enum.h"
+#include "http_field.h"
#include "http_inspect.h"
#include "http_module.h"
#include "http_msg_header.h"
return true;
}
- // If there is no more data to process we need to wrap up file processing right now
- if ((session_data->file_depth_remaining[source_id] > 0) &&
- (session_data->cutter[source_id] != nullptr) &&
+ // If there is no more data to process we may need to tell other components
+ if ((session_data->cutter[source_id] != nullptr) &&
(session_data->cutter[source_id]->get_octets_seen() ==
session_data->partial_raw_bytes[source_id]))
{
- Packet* packet = DetectionEngine::get_current_packet();
- if (!session_data->mime_state[source_id])
+ // Wrap up file processing
+ if (session_data->file_depth_remaining[source_id] > 0)
{
- FileFlows* file_flows = FileFlows::get_file_flows(flow);
- if (!file_flows)
- return false;
-
- const FileDirection dir = (source_id == SRC_SERVER) ? FILE_DOWNLOAD : FILE_UPLOAD;
-
- assert(session_data->transaction[source_id] != nullptr);
- HttpMsgHeader* header = session_data->transaction[source_id]->get_header(source_id);
- assert(header);
-
- uint64_t file_index = header->get_file_cache_index();
- const uint64_t file_processing_id = header->get_multi_file_processing_id();
- file_flows->file_process(packet, file_index, nullptr, 0,
- session_data->file_octets[source_id], dir, file_processing_id, SNORT_FILE_END);
-#ifdef REG_TEST
- if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+ Packet* packet = DetectionEngine::get_current_packet();
+ if (!session_data->mime_state[source_id])
{
- fprintf(HttpTestManager::get_output_file(),
- "File processing finalization during finish()\n");
- fflush(HttpTestManager::get_output_file());
+ FileFlows* file_flows = FileFlows::get_file_flows(flow);
+ if (!file_flows)
+ return false;
+
+ const FileDirection dir = (source_id == SRC_SERVER) ? FILE_DOWNLOAD :
+ FILE_UPLOAD;
+
+ assert(session_data->transaction[source_id] != nullptr);
+ HttpMsgHeader* header = session_data->transaction[source_id]->
+ get_header(source_id);
+ assert(header);
+
+ uint64_t file_index = header->get_file_cache_index();
+ const uint64_t file_processing_id = header->get_multi_file_processing_id();
+ file_flows->file_process(packet, file_index, nullptr, 0,
+ session_data->file_octets[source_id], dir, file_processing_id,
+ SNORT_FILE_END);
+#ifdef REG_TEST
+ if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+ {
+ fprintf(HttpTestManager::get_output_file(),
+ "File processing finalization during finish()\n");
+ fflush(HttpTestManager::get_output_file());
+ }
+#endif
}
+ else
+ {
+ // FIXIT-M The following call does not actually accomplish anything. The MIME
+ // interface needs to be enhanced so that we can communicate end-of-data
+ // without side effects.
+ session_data->mime_state[source_id]->process_mime_data(packet, nullptr, 0, true,
+ SNORT_FILE_POSITION_UNKNOWN);
+ delete session_data->mime_state[source_id];
+ session_data->mime_state[source_id] = nullptr;
+#ifdef REG_TEST
+ if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
+ {
+ fprintf(HttpTestManager::get_output_file(),
+ "MIME finalization during finish()\n");
+ fflush(HttpTestManager::get_output_file());
+ }
#endif
+ }
}
- else
+ // If we were publishing a request body need to publish that body is complete
+ if (session_data->publish_depth_remaining[source_id] > 0)
{
- // FIXIT-M The following call does not actually accomplish anything. The MIME interface
- // needs to be enhanced so that we can communicate end-of-data without side effects.
- session_data->mime_state[source_id]->process_mime_data(packet, nullptr, 0, true,
- SNORT_FILE_POSITION_UNKNOWN);
- delete session_data->mime_state[source_id];
- session_data->mime_state[source_id] = nullptr;
+ HttpRequestBodyEvent http_request_body_event(nullptr,
+ session_data->publish_octets[source_id], true, session_data);
+ DataBus::publish(HTTP2_REQUEST_BODY_EVENT_KEY, http_request_body_event, flow);
#ifdef REG_TEST
if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP))
{
- fprintf(HttpTestManager::get_output_file(), "MIME finalization during finish()\n");
+ fprintf(HttpTestManager::get_output_file(),
+ "Request body event published during finish()\n");
fflush(HttpTestManager::get_output_file());
}
#endif
if (session_data == nullptr)
{
- HttpInspect::http_set_flow_data(flow, session_data = new HttpFlowData);
+ HttpInspect::http_set_flow_data(flow, session_data = new HttpFlowData(flow));
HttpModule::increment_peg_counts(PEG_FLOW);
}
#include "service_inspectors/http_inspect/http_flow_data.h"
#include "service_inspectors/http_inspect/http_module.h"
#include "service_inspectors/http_inspect/http_transaction.h"
+#include "service_inspectors/http2_inspect/http2_flow_data.h"
#include <CppUTest/CommandLineTestRunner.h>
#include <CppUTest/TestHarness.h>
uint32_t str_to_hash(const uint8_t *, size_t) { return 0; }
void FlowData::update_allocations(size_t) {}
void FlowData::update_deallocations(size_t) {}
+FlowData* Flow::get_flow_data(uint32_t) const { return nullptr; }
}
+unsigned Http2FlowData::inspector_id = 0;
+uint32_t Http2FlowData::get_processing_stream_id() const { return 0; }
+
THREAD_LOCAL PegCount HttpModule::peg_counts[PEG_COUNT_MAX] = { };
class HttpUnitTestSetup
TEST_GROUP(http_transaction_test)
{
- HttpFlowData* const flow_data = new HttpFlowData;
+ HttpFlowData* const flow_data = new HttpFlowData(nullptr);
SectionType* const section_type = HttpUnitTestSetup::get_section_type(flow_data);
SectionType* const type_expected = HttpUnitTestSetup::get_type_expected(flow_data);
projects / thirdparty / snort3.git / commitdiff
