Merge pull request #2857 in SNORT/snort3 from ~KATHARVE/snort3:h2i_stream_mem to...

Squashed commit of the following:

commit fbbf12946446eadad1d6e643bec3bda1e310ae7d
Author: Katura Harvey <katharve@cisco.com>
Date:   Wed Apr 21 17:02:24 2021 -0400

    http2_inspect: track stream memory incrementally instead of all up front

diff --git a/src/service_inspectors/http2_inspect/http2_enum.h b/src/service_inspectors/http2_inspect/http2_enum.h
index ac2285380428865fbbc39dacd79b30e67d9cb00a..d3d525c9e6cb2aad33da9c046955b4b573a49fe2 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_enum.h
+++ b/src/service_inspectors/http2_inspect/http2_enum.h
@@ -30,6 +30,9 @@ static const int FRAME_HEADER_LENGTH = 9;
 static const uint32_t NO_STREAM_ID = 0xFFFFFFFF;
 static const uint32_t CONCURRENT_STREAMS_LIMIT = 100;
 
+// Perform memory allocation and deallocation tracking for Http2Stream objects in increments of 25
+static const uint32_t STREAM_MEMORY_TRACKING_INCREMENT = 25;
+
 static const uint32_t HTTP2_GID = 121;
 
 // Frame type codes (fourth octet of frame header)

diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.cc b/src/service_inspectors/http2_inspect/http2_flow_data.cc
index a12e0e917092c4adc83dee12ddafdd6dd4ff0590..6dc60a2d67d8b9f833cb7e822136ac82fa511498 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_flow_data.cc
+++ b/src/service_inspectors/http2_inspect/http2_flow_data.cc
@@ -43,6 +43,11 @@ unsigned Http2FlowData::inspector_id = 0;
 uint64_t Http2FlowData::instance_count = 0;
 #endif
 
+// Each stream will have class Http2Stream allocated and a node in streams list
+const size_t Http2FlowData::stream_memory_size = sizeof(std::_List_node<Http2Stream*>) + sizeof(class Http2Stream);
+const size_t Http2FlowData::stream_increment_memory_size = stream_memory_size *
+    STREAM_MEMORY_TRACKING_INCREMENT;
+
 Http2FlowData::Http2FlowData(Flow* flow_) :
     FlowData(inspector_id),
     flow(flow_),
@@ -97,6 +102,10 @@ Http2FlowData::~Http2FlowData()
 
     for (Http2Stream* stream : streams)
         delete stream;
+    // Since stream memory is allocated in blocks of 25, must also deallocate in blocks of 25 to
+    // ensure consistent rounding.
+    while (stream_memory_allocations_tracked > STREAM_MEMORY_TRACKING_INCREMENT)
+        update_stream_memory_deallocations();
 }
 
 HttpFlowData* Http2FlowData::get_hi_flow_data() const
@@ -115,11 +124,24 @@ void Http2FlowData::set_hi_flow_data(HttpFlowData* flow)
 
 size_t Http2FlowData::size_of()
 {
-    // There are MAX_CONCURRENT_STREAMS + 1 (for stream id 0).
-    // Each stream will have class Http2Stream allocated and a node in streams list
-    const size_t max_streams_size = (MAX_CONCURRENT_STREAMS + 1) *
-        (sizeof(std::_List_node<Http2Stream*>) + sizeof(class Http2Stream));
-    return sizeof(*this) + max_streams_size;
+    // Account for memory for 25 concurrent streams up front, plus 1 stream for stream id 0.
+    return sizeof(*this) + stream_increment_memory_size + stream_memory_size +
+        (2 * sizeof(Http2EventGen)) + (2 * sizeof(Http2Infractions));
+}
+
+void Http2FlowData::update_stream_memory_allocations()
+{
+    assert(concurrent_streams > stream_memory_allocations_tracked);
+    assert(concurrent_streams % stream_memory_allocations_tracked == 1);
+    update_allocations(stream_increment_memory_size);
+    stream_memory_allocations_tracked += STREAM_MEMORY_TRACKING_INCREMENT;
+}
+
+void Http2FlowData::update_stream_memory_deallocations()
+{
+    assert(stream_memory_allocations_tracked >= STREAM_MEMORY_TRACKING_INCREMENT);
+    update_deallocations(stream_increment_memory_size);
+    stream_memory_allocations_tracked -= STREAM_MEMORY_TRACKING_INCREMENT;
 }
 
 Http2Stream* Http2FlowData::find_stream(const uint32_t key) const
@@ -191,6 +213,8 @@ Http2Stream* Http2FlowData::get_stream(const uint32_t key, const SourceId source
             concurrent_streams += 1;
             if (concurrent_streams > Http2Module::get_peg_counts(PEG_MAX_CONCURRENT_STREAMS))
                 Http2Module::increment_peg_counts(PEG_MAX_CONCURRENT_STREAMS);
+            if (concurrent_streams > stream_memory_allocations_tracked)
+                update_stream_memory_allocations();
         }
     }
     return stream;

diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h
index 6ac9d555cfc853ab1b90cf80cbb54cc7bc28400a..f37ae3461c793cb864a5fbdc6fabd933856b7d06 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_flow_data.h
+++ b/src/service_inspectors/http2_inspect/http2_flow_data.h
@@ -153,6 +153,7 @@ protected:
     std::list<Http2Stream*> streams;
     uint32_t concurrent_files = 0;
     uint32_t concurrent_streams = 0;
+    uint32_t stream_memory_allocations_tracked = Http2Enums::STREAM_MEMORY_TRACKING_INCREMENT;
     uint32_t max_stream_id[2] = {0, 0};
     bool delete_stream = false;
 
@@ -205,6 +206,11 @@ private:
     // bookkeeping. So H2I needs to update memory allocations and deallocations itself.
     void allocate_hi_memory(HttpFlowData* hi_flow_data);
     void deallocate_hi_memory(HttpFlowData* hi_flow_data);
+    // Memory for streams is tracked in increments of 25 to minimize tracking overhead
+    void update_stream_memory_allocations();
+    void update_stream_memory_deallocations();
+    static const size_t stream_memory_size;
+    static const size_t stream_increment_memory_size;
 };
 
 #endif