From: Tom Peters (thopeter) Date: Tue, 8 Jun 2021 15:43:48 +0000 (+0000) Subject: Merge pull request #2926 in SNORT/snort3 from ~KATHARVE/snort3:hi_zlib_mem_tracking... X-Git-Tag: 3.1.6.0~16 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8846ffc257cc2b54b9c8b44874340973e9b3fd2d;p=thirdparty%2Fsnort3.git Merge pull request #2926 in SNORT/snort3 from ~KATHARVE/snort3:hi_zlib_mem_tracking to master Squashed commit of the following: commit b4eaceae64f5e73979c4faedf183df491044ba62 Author: Katura Harvey Date: Fri Jun 4 16:47:27 2021 -0400 http_inspect: track memory footprint of zlib inflation --- diff --git a/src/service_inspectors/http_inspect/http_cutter.cc b/src/service_inspectors/http_inspect/http_cutter.cc index 098c17c57..848891a7b 100644 --- a/src/service_inspectors/http_inspect/http_cutter.cc +++ b/src/service_inspectors/http_inspect/http_cutter.cc @@ -25,6 +25,7 @@ #include "http_common.h" #include "http_enum.h" +#include "http_flow_data.h" #include "http_module.h" using namespace HttpEnums; @@ -279,8 +280,9 @@ ScanResult HttpHeaderCutter::cut(const uint8_t* buffer, uint32_t length, } HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_, - CompressId compression_) - : accelerated_blocking(accelerated_blocking_), compression(compression_), finder(finder_) + CompressId compression_, HttpFlowData* ssn_data) + : accelerated_blocking(accelerated_blocking_), compression(compression_), finder(finder_), + session_data(ssn_data) { if (accelerated_blocking) { @@ -291,7 +293,8 @@ HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_ compress_stream->zfree = Z_NULL; compress_stream->next_in = Z_NULL; compress_stream->avail_in = 0; - const int window_bits = (compression == CMP_GZIP) ? GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS; + const int window_bits = (compression == CMP_GZIP) ? + GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS; if (inflateInit2(compress_stream, window_bits) != Z_OK) { assert(false); @@ -299,6 +302,9 @@ HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_ delete compress_stream; compress_stream = nullptr; } + else + session_data->update_allocations(session_data->zlib_inflate_memory); + } static const uint8_t inspect_string[] = { '<', '/', 's', 'c', 'r', 'i', 'p', 't', '>' }; @@ -316,6 +322,7 @@ HttpBodyCutter::~HttpBodyCutter() { inflateEnd(compress_stream); delete compress_stream; + session_data->update_deallocations(session_data->zlib_inflate_memory); } } diff --git a/src/service_inspectors/http_inspect/http_cutter.h b/src/service_inspectors/http_inspect/http_cutter.h index 3cf1100b3..ad6b7df71 100644 --- a/src/service_inspectors/http_inspect/http_cutter.h +++ b/src/service_inspectors/http_inspect/http_cutter.h @@ -27,6 +27,8 @@ #include "http_event.h" #include "http_module.h" +class HttpFlowData; + //------------------------------------------------------------------------- // HttpCutter class and subclasses //------------------------------------------------------------------------- @@ -102,7 +104,7 @@ class HttpBodyCutter : public HttpCutter { public: HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder, - HttpEnums::CompressId compression_); + HttpEnums::CompressId compression_, HttpFlowData* ssn_data); ~HttpBodyCutter() override; void soft_reset() override { octets_seen = 0; } @@ -122,6 +124,7 @@ private: const uint8_t* match_string; const uint8_t* match_string_upper; uint8_t string_length; + HttpFlowData* const session_data; }; class HttpBodyClCutter : public HttpBodyCutter @@ -130,8 +133,10 @@ public: HttpBodyClCutter(int64_t expected_length, bool accelerated_blocking, ScriptFinder* finder, - HttpEnums::CompressId compression) : - HttpBodyCutter(accelerated_blocking, finder, compression), remaining(expected_length) + HttpEnums::CompressId compression, + HttpFlowData* ssn_data) : + HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data), + remaining(expected_length) { assert(remaining > 0); } HttpEnums::ScanResult cut(const uint8_t*, uint32_t length, HttpInfractions*, HttpEventGen*, uint32_t flow_target, bool stretch, HttpEnums::H2BodyState) override; @@ -144,8 +149,8 @@ class HttpBodyOldCutter : public HttpBodyCutter { public: HttpBodyOldCutter(bool accelerated_blocking, ScriptFinder* finder, - HttpEnums::CompressId compression) : - HttpBodyCutter(accelerated_blocking, finder, compression) + HttpEnums::CompressId compression, HttpFlowData* ssn_data) : + HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data) {} HttpEnums::ScanResult cut(const uint8_t*, uint32_t, HttpInfractions*, HttpEventGen*, uint32_t flow_target, bool stretch, HttpEnums::H2BodyState) override; @@ -155,8 +160,8 @@ class HttpBodyChunkCutter : public HttpBodyCutter { public: HttpBodyChunkCutter(bool accelerated_blocking, ScriptFinder* finder, - HttpEnums::CompressId compression) : - HttpBodyCutter(accelerated_blocking, finder, compression) + HttpEnums::CompressId compression, HttpFlowData* ssn_data) : + HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data) {} HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length, HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch, @@ -181,11 +186,9 @@ private: class HttpBodyH2Cutter : public HttpBodyCutter { public: - HttpBodyH2Cutter(int64_t expected_length, - bool accelerated_blocking, - ScriptFinder* finder, - HttpEnums::CompressId compression) : - HttpBodyCutter(accelerated_blocking, finder, compression), + HttpBodyH2Cutter(int64_t expected_length, bool accelerated_blocking, ScriptFinder* finder, + HttpEnums::CompressId compression, HttpFlowData* ssn_data) : + HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data), expected_body_length(expected_length) {} HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length, HttpInfractions*, diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index 89a6fc7c1..3e090b35f 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -115,6 +115,7 @@ HttpFlowData::~HttpFlowData() { inflateEnd(compress_stream[k]); delete compress_stream[k]; + update_deallocations(zlib_inflate_memory); } if (mime_state[k] != nullptr) { @@ -164,6 +165,7 @@ void HttpFlowData::half_reset(SourceId source_id) inflateEnd(compress_stream[source_id]); delete compress_stream[source_id]; compress_stream[source_id] = nullptr; + update_deallocations(zlib_inflate_memory); } if (mime_state[source_id] != nullptr) { @@ -207,6 +209,7 @@ void HttpFlowData::trailer_prep(SourceId source_id) inflateEnd(compress_stream[source_id]); delete compress_stream[source_id]; compress_stream[source_id] = nullptr; + update_deallocations(zlib_inflate_memory); } detection_status[source_id] = DET_REACTIVATING; } diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index d56e23455..38fd4ee71 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -53,6 +53,7 @@ public: static void init() { inspector_id = snort::FlowData::create_flow_data_id(); } size_t size_of() override; + friend class HttpBodyCutter; friend class HttpInspect; friend class HttpJsNorm; friend class HttpMsgSection; @@ -144,9 +145,9 @@ private: // *** Inspector => StreamSplitter (facts about the message section that is coming next) HttpEnums::SectionType type_expected[2] = { HttpEnums::SEC_REQUEST, HttpEnums::SEC_STATUS }; uint64_t last_request_was_connect = false; - // length of the data from Content-Length field z_stream* compress_stream[2] = { nullptr, nullptr }; uint64_t zero_nine_expected = 0; + // length of the data from Content-Length field int64_t data_length[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT }; uint32_t section_size_target[2] = { 0, 0 }; HttpEnums::CompressId compression[2] = { HttpEnums::CMP_NONE, HttpEnums::CMP_NONE }; @@ -214,6 +215,11 @@ private: // Transactions with uncleared sections awaiting deletion HttpTransaction* discard_list = nullptr; + + // Memory footprint required by zlib inflation. Formula from https://zlib.net/zlib_tech.html + // Accounts for a 32k sliding window and 11520 bytes of inflate_huft allocations + static const size_t zlib_inflate_memory = (1 << 15) + 1440*2*sizeof(int); + #ifdef REG_TEST static uint64_t instance_count; uint64_t seq_num; diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index c43587d51..31f7e71b2 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -575,6 +575,8 @@ void HttpMsgHeader::setup_encoding_decompression() delete session_data->compress_stream[source_id]; session_data->compress_stream[source_id] = nullptr; } + else + session_data->update_allocations(session_data->zlib_inflate_memory); } void HttpMsgHeader::setup_utf_decoding() diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h index 2258c30a5..0903ac255 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter.h +++ b/src/service_inspectors/http_inspect/http_stream_splitter.h @@ -55,12 +55,13 @@ private: section_type, uint32_t num_flushed, uint32_t num_excess, int32_t num_head_lines, bool is_broken_chunk, uint32_t num_good_chunks, uint32_t octets_seen) const; - HttpCutter* get_cutter(HttpEnums::SectionType type, const HttpFlowData* session) const; + HttpCutter* get_cutter(HttpEnums::SectionType type, HttpFlowData* session) const; void chunk_spray(HttpFlowData* session_data, uint8_t* buffer, const uint8_t* data, unsigned length) const; static void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream, - bool at_start, HttpInfractions* infractions, HttpEventGen* events); + bool at_start, HttpInfractions* infractions, HttpEventGen* events, + HttpFlowData* session_data); HttpInspect* const my_inspector; const HttpCommon::SourceId source_id; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc index ddfbb4d54..213288f4b 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc @@ -101,7 +101,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount, session_data->compression[source_id], session_data->compress_stream[source_id], at_start, session_data->get_infractions(source_id), - session_data->events[source_id]); + session_data->events[source_id], session_data); if ((expected -= skip_amount) == 0) curr_state = CHUNK_DCRLF1; k += skip_amount-1; @@ -131,7 +131,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount, session_data->compression[source_id], session_data->compress_stream[source_id], at_start, session_data->get_infractions(source_id), - session_data->events[source_id]); + session_data->events[source_id], session_data); k += skip_amount-1; break; } @@ -141,7 +141,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream, - bool at_start, HttpInfractions* infractions, HttpEventGen* events) + bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data) { if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE)) { @@ -179,6 +179,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons inflateEnd(compress_stream); delete compress_stream; compress_stream = nullptr; + session_data->update_deallocations(session_data->zlib_inflate_memory); } return; } @@ -195,7 +196,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons // Start over at the beginning decompress_copy(buffer, offset, data, length, compression, compress_stream, false, - infractions, events); + infractions, events, session_data); return; } else @@ -206,6 +207,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons inflateEnd(compress_stream); delete compress_stream; compress_stream = nullptr; + session_data->update_deallocations(session_data->zlib_inflate_memory); // Since we failed to uncompress the data, fall through } } @@ -396,7 +398,7 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total, decompress_copy(buffer, session_data->section_offset[source_id], data, len, session_data->compression[source_id], session_data->compress_stream[source_id], at_start, session_data->get_infractions(source_id), - session_data->events[source_id]); + session_data->events[source_id], session_data); } else { diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc index 2373279b5..7fafbe54e 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc @@ -61,7 +61,7 @@ void HttpStreamSplitter::prepare_flush(HttpFlowData* session_data, uint32_t* flu } HttpCutter* HttpStreamSplitter::get_cutter(SectionType type, - const HttpFlowData* session_data) const + HttpFlowData* session_data) const { switch (type) { @@ -77,23 +77,23 @@ HttpCutter* HttpStreamSplitter::get_cutter(SectionType type, session_data->data_length[source_id], session_data->accelerated_blocking[source_id], my_inspector->script_finder, - session_data->compression[source_id]); + session_data->compression[source_id], session_data); case SEC_BODY_CHUNK: return (HttpCutter*)new HttpBodyChunkCutter( session_data->accelerated_blocking[source_id], my_inspector->script_finder, - session_data->compression[source_id]); + session_data->compression[source_id], session_data); case SEC_BODY_OLD: return (HttpCutter*)new HttpBodyOldCutter( session_data->accelerated_blocking[source_id], my_inspector->script_finder, - session_data->compression[source_id]); + session_data->compression[source_id], session_data); case SEC_BODY_H2: return (HttpCutter*)new HttpBodyH2Cutter( session_data->data_length[source_id], session_data->accelerated_blocking[source_id], my_inspector->script_finder, - session_data->compression[source_id]); + session_data->compression[source_id], session_data); default: assert(false); return nullptr;