Merge pull request #2873 in SNORT/snort3 from ~THOPETER/snort3:memory1 to master

Squashed commit of the following:

commit 787709393819a2729392f2292707cb8503f7d999
Author: Tom Peters <thopeter@cisco.com>
Date:   Fri Apr 30 17:27:24 2021 -0400

    flow: memory tracking updates

diff --git a/src/flow/flow_data.cc b/src/flow/flow_data.cc
index ec0089250f49e9ace5514368b66243c22805dcc1..b14d008487fced1d81d0abe6bf898291843450a5 100644 (file)
--- a/src/flow/flow_data.cc
+++ b/src/flow/flow_data.cc
@@ -50,19 +50,31 @@ FlowData::~FlowData()
         handler->rem_ref();
 
     assert(mem_in_use == 0);
+    assert(net_allocation_calls == 0);
 }
 
 void FlowData::update_allocations(size_t n)
 {
     memory::MemoryCap::update_allocations(n);
-    mem_in_use += n;
+
+    if (n > 0)
+    {
+        mem_in_use += n;
+        net_allocation_calls++;
+    }
 }
 
 void FlowData::update_deallocations(size_t n)
 {
-    assert(mem_in_use >= n);
     memory::MemoryCap::update_deallocations(n);
-    mem_in_use -= n;
+
+    if (n > 0)
+    {
+        assert(mem_in_use >= n);
+        mem_in_use -= n;
+        assert(net_allocation_calls > 0);
+        net_allocation_calls--;
+    }
 }
 
 RuleFlowData::RuleFlowData(unsigned u) :

diff --git a/src/flow/flow_data.h b/src/flow/flow_data.h
index 79d9ee22da3b083e74a30f5b9049f931301fb186..0b883a5120f74afa5b63dd6a9363724f85e00e96 100644 (file)
--- a/src/flow/flow_data.h
+++ b/src/flow/flow_data.h
@@ -39,6 +39,9 @@ public:
     static unsigned create_flow_data_id()
     { return ++flow_data_id; }
 
+    // Allocations and deallocations must balance. It is not enough that the total number of bytes
+    // allocated and deallocated are equal. They must be allocated and deallocated in the same
+    // increments or roundoffs done inside the functions may not balance.
     void update_allocations(size_t);
     void update_deallocations(size_t);
     Inspector* get_handler() { return handler; }
@@ -60,6 +63,7 @@ private:
     static unsigned flow_data_id;
     Inspector* handler;
     size_t mem_in_use = 0;
+    unsigned net_allocation_calls = 0;
     unsigned id;
 };
 

diff --git a/src/memory/dev_notes.txt b/src/memory/dev_notes.txt
index 73bd469dca5077f0c770ebc0d5c6598780f459d5..740f87968ab16a8a18711d87201ea8222089094e 100644 (file)
--- a/src/memory/dev_notes.txt
+++ b/src/memory/dev_notes.txt
@@ -1,33 +1,11 @@
-This directory provides:
-
-- Facilities for tracking and managing memory usage throughout the program
-- ::operator new and ::operator delete replacements that use the above facilities
-
-The memory interface (found in memory_manager.cc) is templated to accept
-two different components: the Cap and the Allocator.
-
-The Allocator is intended to be a thin interface to the underlying
-heap allocation and deallocation functions (i.e. malloc() and free()).
-They are expected to have the same semantics as malloc() and free().
-
-The Cap is intended to provide callbacks for updating memory allocation
-statistics and a callback that attempts to free a given amount of memory.
-
-During allocation, the Interface first calls Cap::free_space().
-Cap::free_space() is intended as a hook for a function which
-attempts to free space if a memory cap is reached, or to do nothing
-otherwise. Cap::free_space() returns a boolean indicating success
-or failure. If it was successful, then Allocator::allocate() is
-called, followed by Cap::update_allocations().
-
-During deallocation the Interface calls Allocator::deallocate(),
-followed by Cap::update_deallocations().
-
-By swapping out the template parameters for Cap and Allocator,
-the Interface user can implement unit tests with no side-effects,
-or change the behavior of the Interface at compile time. By
-default the allocator and cap located in memory_allocator.h and
-memory_cap.h, respectively, are used in the new/delete replacements.
+This directory provides a simple mechanism for implementing a memory cap.
+Modules can use the MemoryCap::update_allocations() and
+update_deallocations() calls to self-report when they allocate or free
+heap memory. If the total memory allocations exceed the configured memory
+cap, flow pruning is done to free up additional memory.
+
+This mechanism is approximate and does not directly reflect the activities
+of the memory allocator or the OOM killer.
 
 TODO:
 

diff --git a/src/memory/memory_cap.h b/src/memory/memory_cap.h
index 691574553c91e518345b84e37785f760b6f607ee..8a170e3769e6a2d7b507692dd1d0f99f8072dc38 100644 (file)
--- a/src/memory/memory_cap.h
+++ b/src/memory/memory_cap.h
@@ -32,6 +32,8 @@ class SO_PUBLIC MemoryCap
 {
 public:
     static bool free_space(size_t);
+    // The following functions perform internal rounding. Allocations and deallocations must be
+    // performed in identical increments or leakage may occur.
     static void update_allocations(size_t);
     static void update_deallocations(size_t);