From: Tom Peters (thopeter) Date: Mon, 4 Apr 2022 17:58:14 +0000 (+0000) Subject: Pull request #3328: US 697558: http_inspect/http2_inspect: reduce holes in high-volum... X-Git-Tag: 3.1.27.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cefb59504db1338af4d8507b758a52738d4bb699;p=thirdparty%2Fsnort3.git Pull request #3328: US 697558: http_inspect/http2_inspect: reduce holes in high-volume objects Merge in SNORT/snort3 from ~MDAGON/snort3:reduce to master Squashed commit of the following: commit 9d73d54ad9e3420c100aced5eaa97b6977b147a4 Author: Maya Dagon Date: Fri Mar 25 13:46:47 2022 -0400 http2_inspect: reduce holes in objects --- diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index be024b1e8..0e79d357e 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -146,7 +146,6 @@ protected: uint32_t frame_data_size[2] = { 0, 0 }; // Used in eval() - bool frame_in_detection = false; Http2ConnectionSettings connection_settings[2]; Http2HpackDecoder hpack_decoder[2]; std::list streams; @@ -154,6 +153,7 @@ protected: uint32_t concurrent_streams = 0; uint32_t stream_memory_allocations_tracked = Http2Enums::STREAM_MEMORY_TRACKING_INCREMENT; uint32_t max_stream_id[2] = {0, 0}; + bool frame_in_detection = false; bool delete_stream = false; // Internal to scan() @@ -172,14 +172,14 @@ protected: Http2DataCutter data_cutter[2]; // Scan signals to reassemble() + uint32_t bytes_scanned[2] = { 0, 0 }; bool payload_discard[2] = { false, false }; - unsigned bytes_scanned[2] = { 0, 0 }; // Used by scan, reassemble and eval to communicate uint8_t frame_type[2] = { Http2Enums::FT__NONE, Http2Enums::FT__NONE }; bool abort_flow[2] = { false, false }; - std::queue frame_lengths[2]; bool processing_partial_header = false; + std::queue frame_lengths[2]; // Internal to reassemble() uint32_t frame_header_offset[2] = { 0, 0 }; diff --git a/src/service_inspectors/http2_inspect/http2_hpack.h b/src/service_inspectors/http2_inspect/http2_hpack.h index 71de32376..bdefb126c 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack.h +++ b/src/service_inspectors/http2_inspect/http2_hpack.h @@ -89,10 +89,10 @@ private: Http2StartLine* start_line; bool pseudo_headers_allowed; uint8_t* decoded_headers = nullptr; // working buffer to store decoded headers - uint32_t decoded_headers_size = 0; Http2FlowData* session_data; Http2EventGen* const events; Http2Infractions* const infractions; + uint32_t decoded_headers_size = 0; const HttpCommon::SourceId source_id; static Http2HpackIntDecode decode_int7; diff --git a/src/service_inspectors/http_inspect/http_cutter.h b/src/service_inspectors/http_inspect/http_cutter.h index f332996a1..024d2e09b 100644 --- a/src/service_inspectors/http_inspect/http_cutter.h +++ b/src/service_inspectors/http_inspect/http_cutter.h @@ -118,12 +118,12 @@ private: const bool accelerated_blocking; uint8_t partial_match = 0; HttpEnums::CompressId compression; - z_stream* compress_stream = nullptr; bool decompress_failed = false; + uint8_t string_length; + z_stream* compress_stream = nullptr; ScriptFinder* const finder; const uint8_t* match_string; const uint8_t* match_string_upper; - uint8_t string_length; }; class HttpBodyClCutter : public HttpBodyCutter diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index e25f2de3d..050acfb8b 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -87,12 +87,6 @@ public: uint32_t get_h2_stream_id() const; private: - // HTTP/2 handling - bool for_http2 = false; - HttpEnums::H2BodyState h2_body_state[2] = { HttpEnums::H2_BODY_NOT_COMPLETE, - HttpEnums::H2_BODY_NOT_COMPLETE }; - uint32_t h2_stream_id = 0; - // Convenience routines void half_reset(HttpCommon::SourceId source_id); void trailer_prep(HttpCommon::SourceId source_id); @@ -144,7 +138,7 @@ private: // *** Inspector => StreamSplitter (facts about the message section that is coming next) HttpEnums::SectionType type_expected[2] = { HttpEnums::SEC_REQUEST, HttpEnums::SEC_STATUS }; - uint64_t last_request_was_connect = false; + bool last_request_was_connect = false; z_stream* compress_stream[2] = { nullptr, nullptr }; uint64_t zero_nine_expected = 0; // length of the data from Content-Length field @@ -190,27 +184,24 @@ private: // *** Transaction management including pipelining static const int MAX_PIPELINE = 100; // requests seen - responses seen <= MAX_PIPELINE + HttpTransaction* transaction[2] = { nullptr, nullptr }; + + // Transactions with uncleared sections awaiting deletion + HttpTransaction* discard_list = nullptr; + HttpTransaction** pipeline = nullptr; int16_t pipeline_front = 0; int16_t pipeline_back = 0; - uint32_t pdu_idx = 0; - uint32_t js_pdu_idx = 0; - bool js_data_lost_once = false; bool pipeline_overflow = false; bool pipeline_underflow = false; - bool add_to_pipeline(HttpTransaction* latest); HttpTransaction* take_from_pipeline(); void delete_pipeline(); - // Transactions with uncleared sections awaiting deletion - HttpTransaction* discard_list = nullptr; - - - // Memory footprint required by zlib inflation. Formula from https://zlib.net/zlib_tech.html - // Accounts for a 32k sliding window and 11520 bytes of inflate_huft allocations - static const size_t zlib_inflate_memory = (1 << 15) + 1440*2*sizeof(int); + bool js_data_lost_once = false; + uint32_t pdu_idx = 0; + uint32_t js_pdu_idx = 0; // *** HttpJsNorm JSIdentifierCtxBase* js_ident_ctx = nullptr; @@ -229,6 +220,12 @@ private: bool cutover_on_clear = false; bool ssl_search_abandoned = false; + // *** HTTP/2 handling + bool for_http2 = false; + uint32_t h2_stream_id = 0; + HttpEnums::H2BodyState h2_body_state[2] = { HttpEnums::H2_BODY_NOT_COMPLETE, + HttpEnums::H2_BODY_NOT_COMPLETE }; + #ifdef REG_TEST static uint64_t instance_count; uint64_t seq_num; diff --git a/src/service_inspectors/http_inspect/http_msg_section.h b/src/service_inspectors/http_inspect/http_msg_section.h index 58d6c39f7..5c59d32d4 100644 --- a/src/service_inspectors/http_inspect/http_msg_section.h +++ b/src/service_inspectors/http_inspect/http_msg_section.h @@ -110,6 +110,7 @@ protected: HttpEnums::VersionId version_id; HttpEnums::MethodId method_id; const bool tcp_close; + bool cleared = false; // Pointers to related message sections in the same transaction HttpMsgRequest* request; @@ -117,7 +118,6 @@ protected: HttpMsgHeader* header[2]; HttpMsgTrailer* trailer[2]; - bool cleared = false; // Convenience methods shared by multiple subclasses void add_infraction(int infraction);