Merge pull request #2926 in SNORT/snort3 from ~KATHARVE/snort3:hi_zlib_mem_tracking...

Squashed commit of the following:

commit b4eaceae64f5e73979c4faedf183df491044ba62
Author: Katura Harvey <katharve@cisco.com>
Date:   Fri Jun 4 16:47:27 2021 -0400

    http_inspect: track memory footprint of zlib inflation

diff --git a/src/service_inspectors/http_inspect/http_cutter.cc b/src/service_inspectors/http_inspect/http_cutter.cc
index 098c17c571f95b71fa0705be217bbc5c6e9ba379..848891a7b84e3e235bea1c62d950483022d2bc17 100644 (file)
--- a/src/service_inspectors/http_inspect/http_cutter.cc
+++ b/src/service_inspectors/http_inspect/http_cutter.cc
@@ -25,6 +25,7 @@
 
 #include "http_common.h"
 #include "http_enum.h"
+#include "http_flow_data.h"
 #include "http_module.h"
 
 using namespace HttpEnums;
@@ -279,8 +280,9 @@ ScanResult HttpHeaderCutter::cut(const uint8_t* buffer, uint32_t length,
 }
 
 HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_,
-    CompressId compression_)
-    : accelerated_blocking(accelerated_blocking_), compression(compression_), finder(finder_)
+    CompressId compression_, HttpFlowData* ssn_data)
+    : accelerated_blocking(accelerated_blocking_), compression(compression_), finder(finder_),
+    session_data(ssn_data)
 {
     if (accelerated_blocking)
     {
@@ -291,7 +293,8 @@ HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_
             compress_stream->zfree = Z_NULL;
             compress_stream->next_in = Z_NULL;
             compress_stream->avail_in = 0;
-            const int window_bits = (compression == CMP_GZIP) ? GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS;
+            const int window_bits = (compression == CMP_GZIP) ?
+                GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS;
             if (inflateInit2(compress_stream, window_bits) != Z_OK)
             {
                 assert(false);
@@ -299,6 +302,9 @@ HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_
                 delete compress_stream;
                 compress_stream = nullptr;
             }
+            else
+                session_data->update_allocations(session_data->zlib_inflate_memory);
+
         }
 
         static const uint8_t inspect_string[] = { '<', '/', 's', 'c', 'r', 'i', 'p', 't', '>' };
@@ -316,6 +322,7 @@ HttpBodyCutter::~HttpBodyCutter()
     {
         inflateEnd(compress_stream);
         delete compress_stream;
+        session_data->update_deallocations(session_data->zlib_inflate_memory);
     }
 }
 

diff --git a/src/service_inspectors/http_inspect/http_cutter.h b/src/service_inspectors/http_inspect/http_cutter.h
index 3cf1100b31e074faebd9c9c0a541a19f62ace5a0..ad6b7df7160c30ee1551014acde38ad55cf7fafb 100644 (file)
--- a/src/service_inspectors/http_inspect/http_cutter.h
+++ b/src/service_inspectors/http_inspect/http_cutter.h
@@ -27,6 +27,8 @@
 #include "http_event.h"
 #include "http_module.h"
 
+class HttpFlowData;
+
 //-------------------------------------------------------------------------
 // HttpCutter class and subclasses
 //-------------------------------------------------------------------------
@@ -102,7 +104,7 @@ class HttpBodyCutter : public HttpCutter
 {
 public:
     HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder,
-        HttpEnums::CompressId compression_);
+        HttpEnums::CompressId compression_, HttpFlowData* ssn_data);
     ~HttpBodyCutter() override;
     void soft_reset() override { octets_seen = 0; }
 
@@ -122,6 +124,7 @@ private:
     const uint8_t* match_string;
     const uint8_t* match_string_upper;
     uint8_t string_length;
+    HttpFlowData* const session_data;
 };
 
 class HttpBodyClCutter : public HttpBodyCutter
@@ -130,8 +133,10 @@ public:
     HttpBodyClCutter(int64_t expected_length,
         bool accelerated_blocking,
         ScriptFinder* finder,
-        HttpEnums::CompressId compression) :
-        HttpBodyCutter(accelerated_blocking, finder, compression), remaining(expected_length)
+        HttpEnums::CompressId compression,
+        HttpFlowData* ssn_data) :
+        HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data),
+        remaining(expected_length)
         { assert(remaining > 0); }
     HttpEnums::ScanResult cut(const uint8_t*, uint32_t length, HttpInfractions*, HttpEventGen*,
         uint32_t flow_target, bool stretch, HttpEnums::H2BodyState) override;
@@ -144,8 +149,8 @@ class HttpBodyOldCutter : public HttpBodyCutter
 {
 public:
     HttpBodyOldCutter(bool accelerated_blocking, ScriptFinder* finder,
-        HttpEnums::CompressId compression) :
-        HttpBodyCutter(accelerated_blocking, finder, compression)
+        HttpEnums::CompressId compression, HttpFlowData* ssn_data) :
+        HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data)
         {}
     HttpEnums::ScanResult cut(const uint8_t*, uint32_t, HttpInfractions*, HttpEventGen*,
         uint32_t flow_target, bool stretch, HttpEnums::H2BodyState) override;
@@ -155,8 +160,8 @@ class HttpBodyChunkCutter : public HttpBodyCutter
 {
 public:
     HttpBodyChunkCutter(bool accelerated_blocking, ScriptFinder* finder,
-        HttpEnums::CompressId compression) :
-        HttpBodyCutter(accelerated_blocking, finder, compression)
+        HttpEnums::CompressId compression, HttpFlowData* ssn_data) :
+        HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data)
         {}
     HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length,
         HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch,
@@ -181,11 +186,9 @@ private:
 class HttpBodyH2Cutter : public HttpBodyCutter
 {
 public:
-    HttpBodyH2Cutter(int64_t expected_length,
-        bool accelerated_blocking,
-        ScriptFinder* finder,
-        HttpEnums::CompressId compression) :
-        HttpBodyCutter(accelerated_blocking, finder, compression),
+    HttpBodyH2Cutter(int64_t expected_length, bool accelerated_blocking, ScriptFinder* finder,
+        HttpEnums::CompressId compression, HttpFlowData* ssn_data) :
+        HttpBodyCutter(accelerated_blocking, finder, compression, ssn_data),
             expected_body_length(expected_length)
         {}
     HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length, HttpInfractions*,

diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc
index 89a6fc7c1d53eec3900f2d117568ce3e075ccd7d..3e090b35f29b73971a6a49bcf28a1ca4a475fff8 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.cc
+++ b/src/service_inspectors/http_inspect/http_flow_data.cc
@@ -115,6 +115,7 @@ HttpFlowData::~HttpFlowData()
         {
             inflateEnd(compress_stream[k]);
             delete compress_stream[k];
+            update_deallocations(zlib_inflate_memory);
         }
         if (mime_state[k] != nullptr)
         {
@@ -164,6 +165,7 @@ void HttpFlowData::half_reset(SourceId source_id)
         inflateEnd(compress_stream[source_id]);
         delete compress_stream[source_id];
         compress_stream[source_id] = nullptr;
+        update_deallocations(zlib_inflate_memory);
     }
     if (mime_state[source_id] != nullptr)
     {
@@ -207,6 +209,7 @@ void HttpFlowData::trailer_prep(SourceId source_id)
         inflateEnd(compress_stream[source_id]);
         delete compress_stream[source_id];
         compress_stream[source_id] = nullptr;
+        update_deallocations(zlib_inflate_memory);
     }
     detection_status[source_id] = DET_REACTIVATING;
 }

diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index d56e234557bafbf96929deda0f22732b0c5ae8e1..38fd4ee71748b78989df7fb967919d6f9308dff3 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -53,6 +53,7 @@ public:
     static void init() { inspector_id = snort::FlowData::create_flow_data_id(); }
     size_t size_of() override;
 
+    friend class HttpBodyCutter;
     friend class HttpInspect;
     friend class HttpJsNorm;
     friend class HttpMsgSection;
@@ -144,9 +145,9 @@ private:
     // *** Inspector => StreamSplitter (facts about the message section that is coming next)
     HttpEnums::SectionType type_expected[2] = { HttpEnums::SEC_REQUEST, HttpEnums::SEC_STATUS };
     uint64_t last_request_was_connect = false;
-    // length of the data from Content-Length field
     z_stream* compress_stream[2] = { nullptr, nullptr };
     uint64_t zero_nine_expected = 0;
+    // length of the data from Content-Length field
     int64_t data_length[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
     uint32_t section_size_target[2] = { 0, 0 };
     HttpEnums::CompressId compression[2] = { HttpEnums::CMP_NONE, HttpEnums::CMP_NONE };
@@ -214,6 +215,11 @@ private:
     // Transactions with uncleared sections awaiting deletion
     HttpTransaction* discard_list = nullptr;
 
+
+    // Memory footprint required by zlib inflation. Formula from https://zlib.net/zlib_tech.html
+    // Accounts for a 32k sliding window and 11520 bytes of inflate_huft allocations
+    static const size_t zlib_inflate_memory = (1 << 15) + 1440*2*sizeof(int);
+
 #ifdef REG_TEST
     static uint64_t instance_count;
     uint64_t seq_num;

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index c43587d5130e00c43d75931f8a0f21532981e757..31f7e71b2a5dfe2e6470bb4e9079f88170bd5f5a 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -575,6 +575,8 @@ void HttpMsgHeader::setup_encoding_decompression()
         delete session_data->compress_stream[source_id];
         session_data->compress_stream[source_id] = nullptr;
     }
+    else
+        session_data->update_allocations(session_data->zlib_inflate_memory);
 }
 
 void HttpMsgHeader::setup_utf_decoding()

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h
index 2258c30a5d7ace985eb528cbf247ea34eed05484..0903ac25569ed8953b1d6086cb087a8e1f068c12 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter.h
+++ b/src/service_inspectors/http_inspect/http_stream_splitter.h
@@ -55,12 +55,13 @@ private:
         section_type, uint32_t num_flushed, uint32_t num_excess, int32_t num_head_lines,
         bool is_broken_chunk, uint32_t num_good_chunks, uint32_t octets_seen)
         const;
-    HttpCutter* get_cutter(HttpEnums::SectionType type, const HttpFlowData* session) const;
+    HttpCutter* get_cutter(HttpEnums::SectionType type, HttpFlowData* session) const;
     void chunk_spray(HttpFlowData* session_data, uint8_t* buffer, const uint8_t* data,
         unsigned length) const;
     static void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
         uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
-        bool at_start, HttpInfractions* infractions, HttpEventGen* events);
+        bool at_start, HttpInfractions* infractions, HttpEventGen* events,
+        HttpFlowData* session_data);
 
     HttpInspect* const my_inspector;
     const HttpCommon::SourceId source_id;

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
index ddfbb4d54cc7478201fe42a8fb8b679a9cf50fee..213288f4b1e0914190cd4fb9daabb9b804221220 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
@@ -101,7 +101,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer
             decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount,
                 session_data->compression[source_id], session_data->compress_stream[source_id],
                 at_start, session_data->get_infractions(source_id),
-                session_data->events[source_id]);
+                session_data->events[source_id], session_data);
             if ((expected -= skip_amount) == 0)
                 curr_state = CHUNK_DCRLF1;
             k += skip_amount-1;
@@ -131,7 +131,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer
             decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount,
                 session_data->compression[source_id], session_data->compress_stream[source_id],
                 at_start, session_data->get_infractions(source_id),
-                session_data->events[source_id]);
+                session_data->events[source_id], session_data);
             k += skip_amount-1;
             break;
           }
@@ -141,7 +141,7 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer
 
 void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
     uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
-    bool at_start, HttpInfractions* infractions, HttpEventGen* events)
+    bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data)
 {
     if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE))
     {
@@ -179,6 +179,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons
                 inflateEnd(compress_stream);
                 delete compress_stream;
                 compress_stream = nullptr;
+                session_data->update_deallocations(session_data->zlib_inflate_memory);
             }
             return;
         }
@@ -195,7 +196,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons
 
             // Start over at the beginning
             decompress_copy(buffer, offset, data, length, compression, compress_stream, false,
-                infractions, events);
+                infractions, events, session_data);
             return;
         }
         else
@@ -206,6 +207,7 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons
             inflateEnd(compress_stream);
             delete compress_stream;
             compress_stream = nullptr;
+            session_data->update_deallocations(session_data->zlib_inflate_memory);
             // Since we failed to uncompress the data, fall through
         }
     }
@@ -396,7 +398,7 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total,
         decompress_copy(buffer, session_data->section_offset[source_id], data, len,
             session_data->compression[source_id], session_data->compress_stream[source_id],
             at_start, session_data->get_infractions(source_id),
-            session_data->events[source_id]);
+            session_data->events[source_id], session_data);
     }
     else
     {

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
index 2373279b5ccadbfcef1d2a13f094a00c43c9ac37..7fafbe54e3d0830478a903501e4f29764fbbb77b 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
@@ -61,7 +61,7 @@ void HttpStreamSplitter::prepare_flush(HttpFlowData* session_data, uint32_t* flu
 }
 
 HttpCutter* HttpStreamSplitter::get_cutter(SectionType type,
-    const HttpFlowData* session_data) const
+    HttpFlowData* session_data) const
 {
     switch (type)
     {
@@ -77,23 +77,23 @@ HttpCutter* HttpStreamSplitter::get_cutter(SectionType type,
             session_data->data_length[source_id],
             session_data->accelerated_blocking[source_id],
             my_inspector->script_finder,
-            session_data->compression[source_id]);
+            session_data->compression[source_id], session_data);
     case SEC_BODY_CHUNK:
         return (HttpCutter*)new HttpBodyChunkCutter(
             session_data->accelerated_blocking[source_id],
             my_inspector->script_finder,
-            session_data->compression[source_id]);
+            session_data->compression[source_id], session_data);
     case SEC_BODY_OLD:
         return (HttpCutter*)new HttpBodyOldCutter(
             session_data->accelerated_blocking[source_id],
             my_inspector->script_finder,
-            session_data->compression[source_id]);
+            session_data->compression[source_id], session_data);
     case SEC_BODY_H2:
         return (HttpCutter*)new HttpBodyH2Cutter(
             session_data->data_length[source_id],
             session_data->accelerated_blocking[source_id],
             my_inspector->script_finder,
-            session_data->compression[source_id]);
+            session_data->compression[source_id], session_data);
     default:
         assert(false);
         return nullptr;