From: Mike Stepanek (mstepane) Date: Tue, 2 Jun 2020 14:31:05 +0000 (+0000) Subject: Merge pull request #2236 in SNORT/snort3 from ~KATHARVE/snort3:h2i_hi_memory to master X-Git-Tag: 3.0.1-5~37 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ba521ad5419313a3a624f71b4ed80a31b414bed4;p=thirdparty%2Fsnort3.git Merge pull request #2236 in SNORT/snort3 from ~KATHARVE/snort3:h2i_hi_memory to master Squashed commit of the following: commit a3742b47d9b0437fde14014241e933e3bc1908af Author: Katura Harvey Date: Mon Jun 1 13:27:50 2020 -0400 http2_inspect: track memory usage for http_inspect flows in http2_inspect --- diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.cc b/src/service_inspectors/http2_inspect/http2_flow_data.cc index 9627846eb..0a559b58a 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.cc +++ b/src/service_inspectors/http2_inspect/http2_flow_data.cc @@ -158,3 +158,12 @@ uint32_t Http2FlowData::get_current_stream_id(const HttpCommon::SourceId source_ return current_stream[source_id]; } +void Http2FlowData::allocate_hi_memory() +{ + update_allocations(HttpFlowData::get_memory_usage_estimate()); +} + +void Http2FlowData::deallocate_hi_memory() +{ + update_deallocations(HttpFlowData::get_memory_usage_estimate()); +} diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index 46dd2b1fd..974f2d050 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -165,6 +165,11 @@ private: class Http2Stream* get_stream(uint32_t key); class Http2Stream* get_hi_stream() const; class Http2Stream* find_stream(uint32_t key) const; + + // When H2I allocates http_inspect flows, it bypasses the usual FlowData memory allocation + // bookkeeping. So H2I needs to update memory allocations and deallocations itself. + void allocate_hi_memory(); + void deallocate_hi_memory(); }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_stream.cc b/src/service_inspectors/http2_inspect/http2_stream.cc index 3e6c45378..2ea72a070 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.cc +++ b/src/service_inspectors/http2_inspect/http2_stream.cc @@ -40,6 +40,8 @@ Http2Stream::Http2Stream(uint32_t stream_id_, Http2FlowData* session_data_) : Http2Stream::~Http2Stream() { delete current_frame; + if (hi_flow_data) + session_data->deallocate_hi_memory(); delete hi_flow_data; delete data_cutter[SRC_CLIENT]; delete data_cutter[SRC_SERVER]; @@ -62,6 +64,13 @@ void Http2Stream::clear_frame() current_frame = nullptr; } +void Http2Stream::set_hi_flow_data(HttpFlowData* flow_data) +{ + assert(hi_flow_data == nullptr); + hi_flow_data = flow_data; + session_data->allocate_hi_memory(); +} + const Field& Http2Stream::get_buf(unsigned id) { if (current_frame != nullptr) diff --git a/src/service_inspectors/http2_inspect/http2_stream.h b/src/service_inspectors/http2_inspect/http2_stream.h index bbbf7f0cc..ca77104b4 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.h +++ b/src/service_inspectors/http2_inspect/http2_stream.h @@ -42,8 +42,7 @@ public: void clear_frame(); const Field& get_buf(unsigned id); HttpFlowData* get_hi_flow_data() const { return hi_flow_data; } - void set_hi_flow_data(HttpFlowData* flow_data) - { assert(hi_flow_data == nullptr); hi_flow_data = flow_data; } + void set_hi_flow_data(HttpFlowData* flow_data); HttpMsgSection* get_hi_msg_section() const { return hi_msg_section; } void set_hi_msg_section(HttpMsgSection* section) { hi_msg_section = section; } uint32_t get_xtradata_mask() { return (current_frame != nullptr) ? diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index 04f4fb31d..23ed128a7 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -29,8 +29,12 @@ #include "http_common.h" #include "http_enum.h" #include "http_module.h" +#include "http_msg_header.h" +#include "http_msg_request.h" +#include "http_msg_status.h" #include "http_test_manager.h" #include "http_transaction.h" +#include "http_uri.h" using namespace snort; using namespace HttpCommon; @@ -42,6 +46,10 @@ unsigned HttpFlowData::inspector_id = 0; uint64_t HttpFlowData::instance_count = 0; #endif +const uint16_t HttpFlowData::memory_usage_estimate = sizeof(HttpFlowData) + sizeof(HttpTransaction) + + sizeof(HttpMsgRequest) + sizeof(HttpMsgStatus) + (2 * sizeof(HttpMsgHeader)) + sizeof(HttpUri) + + header_size_estimate + small_things; + HttpFlowData::HttpFlowData() : FlowData(inspector_id) { #ifdef REG_TEST @@ -235,6 +243,11 @@ HttpInfractions* HttpFlowData::get_infractions(SourceId source_id) return transaction[source_id]->get_infractions(source_id); } +uint16_t HttpFlowData::get_memory_usage_estimate() +{ + return memory_usage_estimate; +} + #ifdef REG_TEST void HttpFlowData::show(FILE* out_file) const { diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index d057ae280..2a9be9d1a 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -77,6 +77,8 @@ public: void reset_partial_flush(HttpCommon::SourceId source_id) { partial_flush[source_id] = false; } + static uint16_t get_memory_usage_estimate(); + private: // HTTP/2 handling bool for_http2 = false; @@ -182,6 +184,12 @@ private: // Transactions with uncleared sections awaiting deletion HttpTransaction* discard_list = nullptr; + // Estimates of how much memory http_inspect uses to process a stream for H2I + static const uint16_t header_size_estimate = 3000; // combined raw size of request and + //response message headers + static const uint16_t small_things = 400; // minor memory costs not otherwise accounted for + static const uint16_t memory_usage_estimate; + #ifdef REG_TEST static uint64_t instance_count; uint64_t seq_num;