From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Mon, 26 Apr 2021 19:32:26 +0000 (+0000)
Subject: Merge pull request #2857 in SNORT/snort3 from ~KATHARVE/snort3:h2i_stream_mem to... 
X-Git-Tag: 3.1.5.0~30
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f03e83d7a1a1f02d45ee8f0428447bc61af1ca11;p=thirdparty%2Fsnort3.git

Merge pull request #2857 in SNORT/snort3 from ~KATHARVE/snort3:h2i_stream_mem to master

Squashed commit of the following:

commit fbbf12946446eadad1d6e643bec3bda1e310ae7d
Author: Katura Harvey <katharve@cisco.com>
Date:   Wed Apr 21 17:02:24 2021 -0400

    http2_inspect: track stream memory incrementally instead of all up front
---

diff --git a/src/service_inspectors/http2_inspect/http2_enum.h b/src/service_inspectors/http2_inspect/http2_enum.h
index ac2285380..d3d525c9e 100644
--- a/src/service_inspectors/http2_inspect/http2_enum.h
+++ b/src/service_inspectors/http2_inspect/http2_enum.h
@@ -30,6 +30,9 @@ static const int FRAME_HEADER_LENGTH = 9;
 static const uint32_t NO_STREAM_ID = 0xFFFFFFFF;
 static const uint32_t CONCURRENT_STREAMS_LIMIT = 100;
 
+// Perform memory allocation and deallocation tracking for Http2Stream objects in increments of 25
+static const uint32_t STREAM_MEMORY_TRACKING_INCREMENT = 25;
+
 static const uint32_t HTTP2_GID = 121;
 
 // Frame type codes (fourth octet of frame header)
diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.cc b/src/service_inspectors/http2_inspect/http2_flow_data.cc
index a12e0e917..6dc60a2d6 100644
--- a/src/service_inspectors/http2_inspect/http2_flow_data.cc
+++ b/src/service_inspectors/http2_inspect/http2_flow_data.cc
@@ -43,6 +43,11 @@ unsigned Http2FlowData::inspector_id = 0;
 uint64_t Http2FlowData::instance_count = 0;
 #endif
 
+// Each stream will have class Http2Stream allocated and a node in streams list
+const size_t Http2FlowData::stream_memory_size = sizeof(std::_List_node<Http2Stream*>) + sizeof(class Http2Stream);
+const size_t Http2FlowData::stream_increment_memory_size = stream_memory_size *
+    STREAM_MEMORY_TRACKING_INCREMENT;
+
 Http2FlowData::Http2FlowData(Flow* flow_) :
     FlowData(inspector_id),
     flow(flow_),
@@ -97,6 +102,10 @@ Http2FlowData::~Http2FlowData()
 
     for (Http2Stream* stream : streams)
         delete stream;
+    // Since stream memory is allocated in blocks of 25, must also deallocate in blocks of 25 to
+    // ensure consistent rounding.
+    while (stream_memory_allocations_tracked > STREAM_MEMORY_TRACKING_INCREMENT)
+        update_stream_memory_deallocations();
 }
 
 HttpFlowData* Http2FlowData::get_hi_flow_data() const
@@ -115,11 +124,24 @@ void Http2FlowData::set_hi_flow_data(HttpFlowData* flow)
 
 size_t Http2FlowData::size_of()
 {
-    // There are MAX_CONCURRENT_STREAMS + 1 (for stream id 0).
-    // Each stream will have class Http2Stream allocated and a node in streams list
-    const size_t max_streams_size = (MAX_CONCURRENT_STREAMS + 1) *
-        (sizeof(std::_List_node<Http2Stream*>) + sizeof(class Http2Stream));
-    return sizeof(*this) + max_streams_size;
+    // Account for memory for 25 concurrent streams up front, plus 1 stream for stream id 0.
+    return sizeof(*this) + stream_increment_memory_size + stream_memory_size +
+        (2 * sizeof(Http2EventGen)) + (2 * sizeof(Http2Infractions));
+}
+
+void Http2FlowData::update_stream_memory_allocations()
+{
+    assert(concurrent_streams > stream_memory_allocations_tracked);
+    assert(concurrent_streams % stream_memory_allocations_tracked == 1);
+    update_allocations(stream_increment_memory_size);
+    stream_memory_allocations_tracked += STREAM_MEMORY_TRACKING_INCREMENT;
+}
+
+void Http2FlowData::update_stream_memory_deallocations()
+{
+    assert(stream_memory_allocations_tracked >= STREAM_MEMORY_TRACKING_INCREMENT);
+    update_deallocations(stream_increment_memory_size);
+    stream_memory_allocations_tracked -= STREAM_MEMORY_TRACKING_INCREMENT;
 }
 
 Http2Stream* Http2FlowData::find_stream(const uint32_t key) const
@@ -191,6 +213,8 @@ Http2Stream* Http2FlowData::get_stream(const uint32_t key, const SourceId source
             concurrent_streams += 1;
             if (concurrent_streams > Http2Module::get_peg_counts(PEG_MAX_CONCURRENT_STREAMS))
                 Http2Module::increment_peg_counts(PEG_MAX_CONCURRENT_STREAMS);
+            if (concurrent_streams > stream_memory_allocations_tracked)
+                update_stream_memory_allocations();
         }
     }
     return stream;
diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h
index 6ac9d555c..f37ae3461 100644
--- a/src/service_inspectors/http2_inspect/http2_flow_data.h
+++ b/src/service_inspectors/http2_inspect/http2_flow_data.h
@@ -153,6 +153,7 @@ protected:
     std::list<Http2Stream*> streams;
     uint32_t concurrent_files = 0;
     uint32_t concurrent_streams = 0;
+    uint32_t stream_memory_allocations_tracked = Http2Enums::STREAM_MEMORY_TRACKING_INCREMENT;
     uint32_t max_stream_id[2] = {0, 0};
     bool delete_stream = false;
 
@@ -205,6 +206,11 @@ private:
     // bookkeeping. So H2I needs to update memory allocations and deallocations itself.
     void allocate_hi_memory(HttpFlowData* hi_flow_data);
     void deallocate_hi_memory(HttpFlowData* hi_flow_data);
+    // Memory for streams is tracked in increments of 25 to minimize tracking overhead
+    void update_stream_memory_allocations();
+    void update_stream_memory_deallocations();
+    static const size_t stream_memory_size;
+    static const size_t stream_increment_memory_size;
 };
 
 #endif