From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Fri, 7 May 2021 16:15:23 +0000 (+0000)
Subject: Merge pull request #2873 in SNORT/snort3 from ~THOPETER/snort3:memory1 to master
X-Git-Tag: 3.1.5.0~13
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b945f03674385c0c3a70224603e4560b69464045;p=thirdparty%2Fsnort3.git

Merge pull request #2873 in SNORT/snort3 from ~THOPETER/snort3:memory1 to master

Squashed commit of the following:

commit 787709393819a2729392f2292707cb8503f7d999
Author: Tom Peters <thopeter@cisco.com>
Date:   Fri Apr 30 17:27:24 2021 -0400

    flow: memory tracking updates
---

diff --git a/src/flow/flow_data.cc b/src/flow/flow_data.cc
index ec0089250..b14d00848 100644
--- a/src/flow/flow_data.cc
+++ b/src/flow/flow_data.cc
@@ -50,19 +50,31 @@ FlowData::~FlowData()
         handler->rem_ref();
 
     assert(mem_in_use == 0);
+    assert(net_allocation_calls == 0);
 }
 
 void FlowData::update_allocations(size_t n)
 {
     memory::MemoryCap::update_allocations(n);
-    mem_in_use += n;
+
+    if (n > 0)
+    {
+        mem_in_use += n;
+        net_allocation_calls++;
+    }
 }
 
 void FlowData::update_deallocations(size_t n)
 {
-    assert(mem_in_use >= n);
     memory::MemoryCap::update_deallocations(n);
-    mem_in_use -= n;
+
+    if (n > 0)
+    {
+        assert(mem_in_use >= n);
+        mem_in_use -= n;
+        assert(net_allocation_calls > 0);
+        net_allocation_calls--;
+    }
 }
 
 RuleFlowData::RuleFlowData(unsigned u) :
diff --git a/src/flow/flow_data.h b/src/flow/flow_data.h
index 79d9ee22d..0b883a512 100644
--- a/src/flow/flow_data.h
+++ b/src/flow/flow_data.h
@@ -39,6 +39,9 @@ public:
     static unsigned create_flow_data_id()
     { return ++flow_data_id; }
 
+    // Allocations and deallocations must balance. It is not enough that the total number of bytes
+    // allocated and deallocated are equal. They must be allocated and deallocated in the same
+    // increments or roundoffs done inside the functions may not balance.
     void update_allocations(size_t);
     void update_deallocations(size_t);
     Inspector* get_handler() { return handler; }
@@ -60,6 +63,7 @@ private:
     static unsigned flow_data_id;
     Inspector* handler;
     size_t mem_in_use = 0;
+    unsigned net_allocation_calls = 0;
     unsigned id;
 };
 
diff --git a/src/memory/dev_notes.txt b/src/memory/dev_notes.txt
index 73bd469dc..740f87968 100644
--- a/src/memory/dev_notes.txt
+++ b/src/memory/dev_notes.txt
@@ -1,33 +1,11 @@
-This directory provides:
-
-- Facilities for tracking and managing memory usage throughout the program
-- ::operator new and ::operator delete replacements that use the above facilities
-
-The memory interface (found in memory_manager.cc) is templated to accept
-two different components: the Cap and the Allocator.
-
-The Allocator is intended to be a thin interface to the underlying
-heap allocation and deallocation functions (i.e. malloc() and free()).
-They are expected to have the same semantics as malloc() and free().
-
-The Cap is intended to provide callbacks for updating memory allocation
-statistics and a callback that attempts to free a given amount of memory.
-
-During allocation, the Interface first calls Cap::free_space().
-Cap::free_space() is intended as a hook for a function which
-attempts to free space if a memory cap is reached, or to do nothing
-otherwise. Cap::free_space() returns a boolean indicating success
-or failure. If it was successful, then Allocator::allocate() is
-called, followed by Cap::update_allocations().
-
-During deallocation the Interface calls Allocator::deallocate(),
-followed by Cap::update_deallocations().
-
-By swapping out the template parameters for Cap and Allocator,
-the Interface user can implement unit tests with no side-effects,
-or change the behavior of the Interface at compile time. By
-default the allocator and cap located in memory_allocator.h and
-memory_cap.h, respectively, are used in the new/delete replacements.
+This directory provides a simple mechanism for implementing a memory cap.
+Modules can use the MemoryCap::update_allocations() and
+update_deallocations() calls to self-report when they allocate or free
+heap memory. If the total memory allocations exceed the configured memory
+cap, flow pruning is done to free up additional memory.
+
+This mechanism is approximate and does not directly reflect the activities
+of the memory allocator or the OOM killer.
 
 TODO:
 
diff --git a/src/memory/memory_cap.h b/src/memory/memory_cap.h
index 691574553..8a170e376 100644
--- a/src/memory/memory_cap.h
+++ b/src/memory/memory_cap.h
@@ -32,6 +32,8 @@ class SO_PUBLIC MemoryCap
 {
 public:
     static bool free_space(size_t);
+    // The following functions perform internal rounding. Allocations and deallocations must be
+    // performed in identical increments or leakage may occur.
     static void update_allocations(size_t);
     static void update_deallocations(size_t);