{
public:
ThreadRegexOffload(unsigned max);
- ~ThreadRegexOffload();
+ ~ThreadRegexOffload() override;
void stop() override;
void set_file_policy(FilePolicyBase* fp) { file_policy = fp; }
FilePolicyBase* get_file_policy() { return file_policy; }
+ size_t size_of() override
+ { return sizeof(*this); }
+
private:
void init_file_context(FileDirection, FileContext*);
FileContext* find_main_file_context(FilePosition, FileDirection, size_t id = 0);
#include "flow/session.h"
#include "framework/data_bus.h"
#include "ips_options/ips_flowbits.h"
+#include "memory/memory_cap.h"
#include "protocols/packet.h"
#include "sfip/sf_ip.h"
#include "utils/bitop.h"
{
if ( handler )
handler->rem_ref();
+
+ assert(mem_in_use == 0);
+}
+
+void FlowData::update_allocations(size_t n)
+{
+ memory::MemoryCap::free_space(n);
+ memory::MemoryCap::update_allocations(n);
+ mem_in_use += n;
+}
+
+void FlowData::update_deallocations(size_t n)
+{
+ assert(mem_in_use >= n);
+ memory::MemoryCap::update_deallocations(n);
+ mem_in_use -= n;
}
+size_t FlowData::size_of()
+{ return 1024; } // FIXIT-H remove this default impl
+
Flow::Flow()
{
memset(this, 0, sizeof(*this));
void Flow::term()
{
- if ( session )
- delete session;
+ if ( !session )
+ return;
- free_flow_data();
+ delete session;
+ session = nullptr;
+
+ assert(!flow_data);
if ( mpls_client.length )
delete[] mpls_client.start;
flow_data->prev = fd;
flow_data = fd;
+
+ // this is after actual allocation so we can't prune beforehand
+ // but if we are that close to the edge we are in trouble anyway
+ // large allocations can be accounted for directly
+ fd->update_allocations(fd->size_of());
+
return 0;
}
fd->prev->next = fd->next;
fd->next->prev = fd->prev;
}
+ fd->update_deallocations(fd->size_of());
delete fd;
}
{
FlowData* tmp = fd;
fd = fd->next;
+ tmp->update_deallocations(tmp->size_of());
delete tmp;
}
flow_data = nullptr;
static unsigned create_flow_data_id()
{ return ++flow_data_id; }
+ void update_allocations(size_t);
+ void update_deallocations(size_t);
+
+ // return fixed size (could be an approx avg)
+ // this must be fixed for life of flow data instance
+ // track significant supplemental allocations with the above updaters
+ virtual size_t size_of() = 0;
+
virtual void handle_expected(Packet*) { }
virtual void handle_retransmit(Packet*) { }
virtual void handle_eof(Packet*) { }
private:
static unsigned flow_data_id;
Inspector* handler;
+ size_t mem_in_use = 0;
unsigned id;
};
#include "hash/zhash.h"
#include "helpers/flag_context.h"
#include "ips_options/ips_flowbits.h"
+#include "memory/memory_cap.h"
#include "packet_io/active.h"
#include "time/packet_time.h"
#include "utils/stats.h"
FlowCache::~FlowCache ()
{
- while ( Flow* flow = (Flow*)hash_table->pop() )
- flow->term();
-
delete uni_head;
delete uni_tail;
link_uni(flow);
}
+ memory::MemoryCap::update_allocations(config.cap_weight);
flow->last_data_seen = timestamp;
return flow;
if ( flow->next )
unlink_uni(flow);
+ memory::MemoryCap::update_deallocations(config.cap_weight);
return hash_table->remove(flow->key);
}
+int FlowCache::retire(Flow* flow)
+{
+ flow->reset(true);
+ flow->term();
+ prune_stats.update(PruneReason::NONE);
+ return remove(flow);
+}
+
unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me)
{
ActiveSuspendContext act_susp;
while ( auto flow = static_cast<Flow*>(hash_table->first()) )
{
- release(flow, PruneReason::NONE);
+ retire(flow);
++retired;
}
+ while ( Flow* flow = (Flow*)hash_table->pop() )
+ flow->term();
+
return retired;
}
private:
void link_uni(snort::Flow*);
int remove(snort::Flow*);
+ int retire(snort::Flow*);
private:
static const unsigned cleanup_flows = 1;
unsigned max_sessions = 0;
unsigned pruning_timeout = 0;
unsigned nominal_timeout = 0;
+ unsigned cap_weight = 0;
};
#endif
already_exiting = true;
initializing = false; // just in case we cut out early
+ memory::MemoryCap::print();
+
term_signals();
IpsManager::global_term(SnortConfig::get_conf());
SFAT_Cleanup();
set ( MEMORY_SOURCES
- memory_allocator.cc
- memory_allocator.h
memory_cap.cc
memory_cap.h
memory_module.cc
memory_module.h
memory_config.h
- memory_manager.cc
prune_handler.cc
prune_handler.h
)
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2016-2019 Cisco and/or its affiliates. All rights reserved.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// memory_allocator.cc author Joel Cornett <jocornet@cisco.com>
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include "memory_allocator.h"
-
-#include <cstdlib>
-
-namespace memory
-{
-
-// FIXIT-L (de)allocate() could be inlined if defined in memory_manager.cc
-void* MemoryAllocator::allocate(size_t n)
-{ return malloc(n); }
-
-void MemoryAllocator::deallocate(void* p)
-{ free(p); }
-
-} // namespace memory
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2016-2019 Cisco and/or its affiliates. All rights reserved.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// memory_allocator.h author Joel Cornett <jocornet@cisco.com>
-
-#ifndef MEMORY_ALLOCATOR_H
-#define MEMORY_ALLOCATOR_H
-
-#include <cstddef>
-
-namespace memory
-{
-
-struct MemoryAllocator
-{
- static void* allocate(size_t);
- static void deallocate(void*);
-};
-
-} // namespace memory
-
-#endif
struct Tracker
{
- size_t used_memory = 0;
-
- uint64_t allocations = 0;
- uint64_t deallocations = 0;
-
void allocate(size_t n)
- {
- used_memory += n;
- ++allocations;
- }
+ { mem_stats.allocated += n; ++mem_stats.allocations; }
void deallocate(size_t n)
- {
- // FIXIT-H If memory is allocated in one thread and passed to another thread,
- // when it is freed, we may have (used_memory < memory_to_be_freed)
- // Following assertion will fail.
- //assert(n <= used_memory);
-
- if(n > used_memory)
- return;
-
- used_memory -= n;
- ++deallocations;
- }
+ { mem_stats.deallocated += n; ++mem_stats.deallocations; }
size_t used() const
{
- return used_memory;
+ if ( mem_stats.allocated < mem_stats.deallocated )
+ {
+ assert(false);
+ return 0;
+ }
+ return mem_stats.allocated - mem_stats.deallocated;
}
-
- constexpr Tracker() = default;
};
static THREAD_LOCAL Tracker s_tracker;
auto used = trk.used();
- if ( requested <= cap - used )
+ if ( used + requested <= cap )
return true;
- while ( requested > cap - used )
+ ++mem_stats.reap_attempts;
+
+ while ( used + requested > cap )
{
handler();
-
- // check if the handler freed any space
auto tmp = trk.used();
+
if ( tmp >= used )
{
- // nope
+ ++mem_stats.reap_failures;
return false;
}
-
used = tmp;
}
-
return true;
}
if ( !thread_cap )
return true;
- const auto& config = *snort::SnortConfig::get_conf()->memory;
- return memory::free_space(n, thread_cap, s_tracker, prune_handler) || config.soft;
+ return memory::free_space(n, thread_cap, s_tracker, prune_handler);
}
+static size_t fudge_it(size_t n)
+{ return ((n >> 7) + 1) << 7; }
+
void MemoryCap::update_allocations(size_t n)
{
+ size_t k = n;
+ n = fudge_it(n);
+ mem_stats.total_fudge += (n - k);
s_tracker.allocate(n);
+ auto in_use = mem_stats.allocated - mem_stats.deallocated;
+ if ( in_use > mem_stats.max_in_use )
+ mem_stats.max_in_use = in_use;
mp_active_context.update_allocs(n);
}
void MemoryCap::update_deallocations(size_t n)
{
+ n = fudge_it(n);
s_tracker.deallocate(n);
mp_active_context.update_deallocs(n);
}
const MemoryConfig& config = *snort::SnortConfig::get_conf()->memory;
- if ( snort::SnortConfig::log_verbose() or s_tracker.allocations )
+ if ( snort::SnortConfig::log_verbose() or mem_stats.allocations )
LogLabel("memory (heap)");
if ( snort::SnortConfig::log_verbose() )
{
LogMessage(" global cap: %zu\n", config.cap);
LogMessage(" global preemptive threshold percent: %u\n", config.threshold);
- LogMessage(" cap type: %s\n", config.soft? "soft" : "hard");
}
- if ( s_tracker.allocations )
+ if ( mem_stats.allocations )
{
LogMessage(" main thread usage: %zu\n", s_tracker.used());
- LogMessage(" allocations: %" PRIu64 "\n", s_tracker.allocations);
- LogMessage(" deallocations: %" PRIu64 "\n", s_tracker.deallocations);
+ LogMessage(" allocations: %" PRIu64 "\n", mem_stats.allocations);
+ LogMessage(" deallocations: %" PRIu64 "\n", mem_stats.deallocations);
LogMessage(" thread cap: %zu\n", thread_cap);
LogMessage(" preemptive threshold: %zu\n", preemptive_threshold);
}
CHECK_FALSE( memory::free_space(1, 1024, tracker, handler) );
CHECK( handler.calls == 1 );
}
-
- SECTION( "free_space() does not rollover" )
- {
- MockTracker tracker { SIZE_MAX };
- HandlerSpy handler { -5, tracker };
- CHECK( memory::free_space(1, SIZE_MAX, tracker, handler) );
- CHECK( handler.calls == 1 );
- }
-
}
#endif
#include <cstddef>
+#include "main/thread.h"
+
namespace memory
{
{
size_t cap = 0;
unsigned threshold = 0;
- bool soft = false;
constexpr MemoryConfig() = default;
};
+++ /dev/null
-//--------------------------------------------------------------------------
-// Copyright (C) 2016-2019 Cisco and/or its affiliates. All rights reserved.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation. You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// memory_manager.cc author Joel Cornett <jocornet@cisco.com>
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <cassert>
-#include <new>
-
-#include "main/thread.h"
-
-#include "memory_allocator.h"
-#include "memory_cap.h"
-
-#ifdef UNIT_TEST
-#include "catch/snort_catch.h"
-#endif
-
-namespace memory
-{
-
-// -----------------------------------------------------------------------------
-// metadata
-// -----------------------------------------------------------------------------
-
-// NOTE: This structure must be aligned to max_align_t as long as we are prepending
-// it to memory allocations so that the returned memory is also aligned.
-struct alignas(max_align_t) Metadata
-{
-#if defined(REG_TEST) || defined(UNIT_TEST)
- static constexpr size_t SANITY_CHECK_VALUE = 0xabcdef;
- size_t sanity;
-#endif
-
- // number of requested bytes
- size_t payload_size;
-
- // total number of bytes allocated, including Metadata header
- size_t total_size() const;
- void* payload_offset();
-
-#if defined(REG_TEST) || defined(UNIT_TEST)
- bool valid() const
- { return sanity == SANITY_CHECK_VALUE; }
-#endif
-
- Metadata(size_t = 0);
-
- static size_t calculate_total_size(size_t);
-
- template<typename Allocator>
- static Metadata* create(size_t);
-
- static Metadata* extract(void*);
-};
-
-inline size_t Metadata::total_size() const
-{ return calculate_total_size(payload_size); }
-
-inline void* Metadata::payload_offset()
-{ return this + 1; }
-
-inline Metadata::Metadata(size_t n) :
-#if defined(REG_TEST) || defined(UNIT_TEST)
- sanity(SANITY_CHECK_VALUE),
-#endif
- payload_size(n)
-{ }
-
-inline size_t Metadata::calculate_total_size(size_t n)
-{ return sizeof(Metadata) + n; }
-
-template<typename Allocator>
-Metadata* Metadata::create(size_t n)
-{
- auto meta =
- static_cast<Metadata*>(Allocator::allocate(calculate_total_size(n)));
-
- if ( !meta )
- return nullptr;
-
- // Trigger metadata ctor
- *meta = Metadata(n);
-
-#if defined(REG_TEST) || defined(UNIT_TEST)
- assert(meta->valid());
-#endif
-
- return meta;
-}
-
-Metadata* Metadata::extract(void* p)
-{
- assert(p);
-
- auto meta = static_cast<Metadata*>(p) - 1;
-
-#if defined(REG_TEST) || defined(UNIT_TEST)
- assert(meta->valid());
-#endif
-
- return meta;
-}
-
-// -----------------------------------------------------------------------------
-// the meat
-// -----------------------------------------------------------------------------
-
-class ReentryContext
-{
-public:
- ReentryContext(bool& flag) :
- already_entered(flag), flag(flag)
- { flag = true; }
-
- ~ReentryContext()
- { flag = false; }
-
- bool is_reentry() const
- { return already_entered; }
-
-private:
- const bool already_entered;
- bool& flag;
-};
-
-template<typename Allocator = MemoryAllocator, typename Cap = MemoryCap>
-struct Interface
-{
- static void* allocate(size_t);
- static void deallocate(void*);
-
- static THREAD_LOCAL bool in_allocation_call;
-};
-
-template<typename Allocator, typename Cap>
-void* Interface<Allocator, Cap>::allocate(size_t n)
-{
- // prevent allocation reentry
- ReentryContext reentry_context(in_allocation_call);
- assert(!reentry_context.is_reentry());
-
- if ( !Cap::free_space(Metadata::calculate_total_size(n)) )
- return nullptr;
-
- auto meta = Metadata::create<Allocator>(n);
- if ( !meta )
- return nullptr;
-
- Cap::update_allocations(meta->total_size());
- return meta->payload_offset();
-}
-
-template<typename Allocator, typename Cap>
-void Interface<Allocator, Cap>::deallocate(void* p)
-{
- if ( !p )
- return;
-
- auto meta = Metadata::extract(p);
- assert(meta);
-
- Cap::update_deallocations(meta->total_size());
- Allocator::deallocate(meta);
-}
-
-template<typename Allocator, typename Cap>
-THREAD_LOCAL bool Interface<Allocator, Cap>::in_allocation_call = false;
-
-} //namespace memory
-
-// -----------------------------------------------------------------------------
-// new /delete replacements
-// -----------------------------------------------------------------------------
-
-// these don't have to be visible to operate as replacements
-
-#ifndef NO_MEM_MGR
-void* operator new(size_t n)
-{
- auto p = memory::Interface<>::allocate(n);
- if ( !p )
- throw std::bad_alloc();
-
- return p;
-}
-
-void* operator new(size_t n, const std::nothrow_t&) noexcept
-{ return memory::Interface<>::allocate(n); }
-
-void operator delete(void* p) noexcept
-{ memory::Interface<>::deallocate(p); }
-
-void operator delete(void* p, const std::nothrow_t&) noexcept
-{ ::operator delete(p); }
-
-void* operator new[](size_t n)
-{ return ::operator new(n); }
-
-void* operator new[](size_t n, const std::nothrow_t& tag) noexcept
-{ return ::operator new(n, tag); }
-
-void operator delete[](void* p) noexcept
-{ ::operator delete(p); }
-
-void operator delete[](void* p, const std::nothrow_t&) noexcept
-{ ::operator delete[](p); }
-
-// C++14 delete operators are a special case and must be explicitly exported
-// since we're compiling as C++11 but must capture these for external libraries
-void operator delete(void* p, size_t) noexcept;
-SO_PUBLIC void operator delete(void* p, size_t) noexcept
-{ ::operator delete(p); }
-
-void operator delete[](void* p, size_t) noexcept;
-SO_PUBLIC void operator delete[](void* p, size_t) noexcept
-{ ::operator delete[](p); }
-#endif
-
-// -----------------------------------------------------------------------------
-// unit tests
-// -----------------------------------------------------------------------------
-
-#ifdef UNIT_TEST
-
-namespace t_memory
-{
-
-struct AllocatorSpy
-{
- static void* allocate(size_t n)
- { allocate_called = true; allocate_arg = n; return pool; }
-
- static void deallocate(void* p)
- { deallocate_called = true; deallocate_arg = p; }
-
- static void reset()
- {
- pool = nullptr;
- allocate_called = false;
- allocate_arg = 0;
- deallocate_called = false;
- deallocate_arg = nullptr;
- }
-
- static void* pool;
- static bool allocate_called;
- static size_t allocate_arg;
- static bool deallocate_called;
- static void* deallocate_arg;
-};
-
-void* AllocatorSpy::pool = nullptr;
-bool AllocatorSpy::allocate_called = false;
-size_t AllocatorSpy::allocate_arg = 0;
-bool AllocatorSpy::deallocate_called = false;
-void* AllocatorSpy::deallocate_arg = nullptr;
-
-struct CapSpy
-{
- static bool free_space(size_t n)
- {
- free_space_called = true;
- free_space_arg = n;
- return free_space_result;
- }
-
- static void update_allocations(size_t n)
- {
- update_allocations_called = true;
- update_allocations_arg = n;
- }
-
- static void update_deallocations(size_t n)
- {
- update_deallocations_called = true;
- update_deallocations_arg = n;
- }
-
- static void reset()
- {
- free_space_called = false;
- free_space_arg = 0;
- free_space_result = false;
-
- update_allocations_called = false;
- update_allocations_arg = 0;
-
- update_deallocations_called = false;
- update_deallocations_arg = 0;
- }
-
- static bool free_space_called;
- static size_t free_space_arg;
- static bool free_space_result;
-
- static bool update_allocations_called;
- static size_t update_allocations_arg;
-
- static bool update_deallocations_called;
- static size_t update_deallocations_arg;
-};
-
-bool CapSpy::free_space_called = false;
-size_t CapSpy::free_space_arg = 0;
-bool CapSpy::free_space_result = false;
-
-bool CapSpy::update_allocations_called = false;
-size_t CapSpy::update_allocations_arg = 0;
-
-bool CapSpy::update_deallocations_called = false;
-size_t CapSpy::update_deallocations_arg = 0;
-
-} // namespace t_memory
-
-TEST_CASE( "memory metadata", "[memory]" )
-{
- using namespace t_memory;
-
- AllocatorSpy::reset();
- constexpr size_t n = 1;
- char pool[sizeof(memory::Metadata) + n];
-
- SECTION( "create" )
- {
- AllocatorSpy::pool = pool;
-
- auto meta = memory::Metadata::create<AllocatorSpy>(n);
-
- CHECK( (void*)meta == (void*)pool );
- CHECK( meta->valid() );
- CHECK( meta->payload_size == n );
- }
-
- SECTION( "extract" )
- {
- auto meta_pool = reinterpret_cast<memory::Metadata*>(pool);
- meta_pool[0] = memory::Metadata(n);
-
- void* p = &meta_pool[1];
-
- auto meta = memory::Metadata::extract(p);
-
- CHECK( (void*)meta == (void*)pool );
- CHECK( meta->payload_offset() == p );
- }
-}
-
-TEST_CASE( "memory manager interface", "[memory]" )
-{
- using namespace t_memory;
-
- AllocatorSpy::reset();
- CapSpy::reset();
-
- constexpr size_t n = 1;
- char pool[sizeof(memory::Metadata) + n];
-
- using Interface = memory::Interface<AllocatorSpy, CapSpy>;
-
- SECTION( "allocation" )
- {
- SECTION( "free space failure" )
- {
- auto p = Interface::allocate(n);
-
- CHECK( p == nullptr );
-
- CHECK( CapSpy::free_space_called );
- CHECK( CapSpy::free_space_arg == memory::Metadata::calculate_total_size(n) );
-
- CHECK_FALSE( AllocatorSpy::allocate_called );
- CHECK_FALSE( CapSpy::update_allocations_called );
- }
-
- SECTION( "allocation failure" )
- {
- CapSpy::free_space_result = true;
-
- auto p = Interface::allocate(n);
-
- CHECK( p == nullptr );
-
- CHECK( CapSpy::free_space_called );
- CHECK( CapSpy::free_space_arg == memory::Metadata::calculate_total_size(n) );
-
- CHECK( AllocatorSpy::allocate_called );
- CHECK( AllocatorSpy::allocate_arg == memory::Metadata::calculate_total_size(n) );
-
- CHECK_FALSE( CapSpy::update_allocations_called );
- }
-
- SECTION( "success" )
- {
- CapSpy::free_space_result = true;
- AllocatorSpy::pool = pool;
-
- auto p = Interface::allocate(n);
-
- CHECK( p > (void*)pool );
-
- CHECK( CapSpy::free_space_called );
- CHECK( CapSpy::free_space_arg == memory::Metadata::calculate_total_size(n) );
-
- CHECK( AllocatorSpy::allocate_called );
- CHECK( AllocatorSpy::allocate_arg == memory::Metadata::calculate_total_size(n) );
-
- CHECK( CapSpy::update_allocations_called );
- CHECK( CapSpy::update_allocations_arg == memory::Metadata::calculate_total_size(n) );
- }
- }
-
- SECTION( "deallocation" )
- {
- SECTION( "nullptr" )
- {
- Interface::deallocate(nullptr);
-
- CHECK_FALSE( AllocatorSpy::deallocate_called );
- CHECK_FALSE( CapSpy::update_deallocations_called );
- }
-
- SECTION( "success" )
- {
- auto meta_pool = reinterpret_cast<memory::Metadata*>(pool);
- meta_pool[0] = memory::Metadata(n);
-
- auto p = meta_pool[0].payload_offset();
-
- Interface::deallocate(p);
-
- CHECK( AllocatorSpy::deallocate_called );
- CHECK( AllocatorSpy::deallocate_arg == (void*)pool );
- CHECK( CapSpy::update_deallocations_called );
- CHECK( CapSpy::update_deallocations_arg == memory::Metadata::calculate_total_size(n) );
- }
- }
- AllocatorSpy::pool = nullptr;
- AllocatorSpy::deallocate_arg = nullptr;
-}
-
-#endif
{ "cap", Parameter::PT_INT, "0:maxSZ", "0",
"set the per-packet-thread cap on memory (bytes, 0 to disable)" },
- { "soft", Parameter::PT_BOOL, nullptr, "false",
- "always succeed in allocating memory, even if above the cap" },
-
{ "threshold", Parameter::PT_INT, "0:100", "0",
"set the per-packet-thread threshold for preemptive cleanup actions "
"(percent, 0 to disable)" },
{ nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
};
+THREAD_LOCAL MemoryCounts mem_stats;
+static MemoryCounts zero_stats = { };
+
+const PegInfo mem_pegs[] =
+{
+ { CountType::SUM, "allocations", "total number of allocations" },
+ { CountType::SUM, "deallocations", "total number of deallocations" },
+ { CountType::SUM, "allocated", "total amount of memory allocated" },
+ { CountType::SUM, "deallocated", "total amount of memory allocated" },
+ { CountType::SUM, "reap_attempts", "attempts to reclaim memory" },
+ { CountType::SUM, "reap_failures", "failures to reclaim memory" },
+ { CountType::MAX, "max_in_use", "highest allocated - deallocated" },
+ { CountType::SUM, "total_fudge", "sum of all adjustments" },
+ { CountType::END, nullptr, nullptr }
+};
+
// -----------------------------------------------------------------------------
// memory module
// -----------------------------------------------------------------------------
if ( v.is("cap") )
sc->memory->cap = v.get_size();
- else if ( v.is("soft") )
- sc->memory->soft = v.get_bool();
-
else if ( v.is("threshold") )
sc->memory->threshold = v.get_uint8();
bool MemoryModule::is_active()
{ return configured; }
+const PegInfo* MemoryModule::get_pegs() const
+{ return mem_pegs; }
+
+PegCount* MemoryModule::get_counts() const
+{ return is_active() ? (PegCount*)&mem_stats : (PegCount*)&zero_stats; }
+
#include "framework/module.h"
+struct MemoryCounts
+{
+ PegCount allocations;
+ PegCount deallocations;
+ PegCount allocated;
+ PegCount deallocated;
+ PegCount reap_attempts;
+ PegCount reap_failures;
+ PegCount max_in_use;
+ PegCount total_fudge;
+};
+
+extern THREAD_LOCAL MemoryCounts mem_stats;
+
class MemoryModule : public snort::Module
{
public:
MemoryModule();
+ const PegInfo* get_pegs() const override;
+ PegCount* get_counts() const override;
+
bool set(const char*, snort::Value&, snort::SnortConfig*) override;
bool end(const char*, int, snort::SnortConfig*) override;
uint16_t, IpProtocol, SnortProtocolId, int, AppIdInspector&);
AppIdInspector& get_inspector() const
- {
- return inspector;
- }
+ { return inspector; }
+
+ size_t size_of() override
+ { return sizeof(*this); }
uint32_t session_id = 0;
snort::Flow* flow = nullptr;
~Dce2SmbFlowData() override;
static void init()
- {
- inspector_id = snort::FlowData::create_flow_data_id();
- }
+ { inspector_id = snort::FlowData::create_flow_data_id(); }
+
+ size_t size_of() override
+ { return sizeof(*this); }
public:
static unsigned inspector_id;
~Dce2TcpFlowData() override;
static void init()
- {
- inspector_id = snort::FlowData::create_flow_data_id();
- }
+ { inspector_id = snort::FlowData::create_flow_data_id(); }
+
+ size_t size_of() override
+ { return sizeof(*this); }
public:
static unsigned inspector_id;
~Dce2UdpFlowData() override;
static void init()
- {
- inspector_id = snort::FlowData::create_flow_data_id();
- }
+ { inspector_id = snort::FlowData::create_flow_data_id(); }
static unsigned inspector_id;
DCE2_UdpSsnData dce2_udp_session;
+
+ size_t size_of() override
+ { return sizeof(*this); }
};
DCE2_UdpSsnData* get_dce2_udp_session_data(snort::Flow*);
~Dnp3FlowData() override;
static void init()
- {
- inspector_id = snort::FlowData::create_flow_data_id();
- }
+ { inspector_id = snort::FlowData::create_flow_data_id(); }
+
+ size_t size_of() override
+ { return sizeof(*this); }
public:
static unsigned inspector_id;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
DNSData session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
TELNET_SESSION session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
FTP_SESSION session;
void handle_expected(snort::Packet*) override;
void handle_eof(snort::Packet*) override;
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
FTP_DATA_SESSION session;
static void init();
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
GTP_Roptions ropts;
friend bool implement_get_buf(unsigned id, Http2FlowData*, Http2Enums::SourceId,
snort::InspectionBuffer&);
+ size_t size_of() override
+ { return sizeof(*this); }
+
protected:
// 0 element refers to client frame, 1 element refers to server frame
bool preface[2] = { true, false };
friend class HttpUnitTestSetup;
#endif
+ size_t size_of() override
+ { return sizeof(*this); }
+
private:
// Convenience routines
void half_reset(HttpEnums::SourceId source_id);
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
IMAPData session;
ssn_data.flags = 0;
}
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
modbus_session_data_t ssn_data;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
POPData session;
static void init()
{ inspector_id = FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
RpcSsnData session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
SIPData session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
SMTPData session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
SSHData session;
static void init()
{ inspector_id = snort::FlowData::create_flow_data_id(); }
+ size_t size_of() override
+ { return sizeof(*this); }
+
public:
static unsigned inspector_id;
SSLData session;
static void base_tterm()
{
+ // this can't happen sooner because the counts haven't been harvested yet
delete flow_con;
flow_con = nullptr;
}
//-------------------------------------------------------------------------
Trace TRACE_NAME(stream);
-#define CACHE_PARAMS(name, max, prune, idle, cleanup) \
+#define CACHE_PARAMS(name, max, prune, idle, weight) \
static const Parameter name[] = \
{ \
{ "max_sessions", Parameter::PT_INT, "2:max32", max, \
\
{ "idle_timeout", Parameter::PT_INT, "1:max32", idle, \
"maximum inactive time before retiring session tracker" }, \
+ \
+ { "cap_weight", Parameter::PT_INT, "0:65535", idle, \
+ "additional bytes to track per flow for better estimation against cap" }, \
\
{ nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } \
}
-CACHE_PARAMS(ip_params, "16384", "30", "180", "5");
-CACHE_PARAMS(icmp_params, "65536", "30", "180", "5");
-CACHE_PARAMS(tcp_params, "262144", "30", "3600", "5");
-CACHE_PARAMS(udp_params, "131072", "30", "180", "5");
-CACHE_PARAMS(user_params, "1024", "30", "180", "5");
-CACHE_PARAMS(file_params, "128", "30", "180", "5");
+CACHE_PARAMS(ip_params, "16384", "30", "180", "64");
+CACHE_PARAMS(icmp_params, "65536", "30", "180", "8");
+CACHE_PARAMS(tcp_params, "262144", "30", "3600", "256");
+CACHE_PARAMS(udp_params, "131072", "30", "180", "128");
+CACHE_PARAMS(user_params, "1024", "30", "180", "256");
+CACHE_PARAMS(file_params, "128", "30", "180", "32");
#define CACHE_TABLE(cache, proto, params) \
{ cache, Parameter::PT_TABLE, params, nullptr, \
else if ( v.is("idle_timeout") )
fc->nominal_timeout = v.get_uint32();
+ else if ( v.is("cap_weight") )
+ fc->cap_weight = v.get_uint16();
+
else
return false;
#include "detection/detection_engine.h"
#include "file_api/file_flows.h"
+#include "memory/memory_cap.h"
#include "packet_io/sfdaq.h"
#include "profiler/profiler_defs.h"
#include "protocols/packet.h"
// FileSession methods
//-------------------------------------------------------------------------
-FileSession::FileSession(Flow* flow) : Session(flow) { }
+FileSession::FileSession(Flow* flow) : Session(flow)
+{ memory::MemoryCap::update_allocations(sizeof(*this)); }
+FileSession::~FileSession()
+{ memory::MemoryCap::update_deallocations(sizeof(*this)); }
bool FileSession::setup(Packet*)
{
{
public:
FileSession(snort::Flow*);
+ ~FileSession() override;
bool setup(snort::Packet*) override;
void clear() override;
#include "icmp_session.h"
#include "flow/flow_key.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler_defs.h"
#include "protocols/icmp4.h"
#include "protocols/packet.h"
//-------------------------------------------------------------------------
IcmpSession::IcmpSession(Flow* flow) : Session(flow)
-{
-}
+{ memory::MemoryCap::update_allocations(sizeof(*this)); }
+
+IcmpSession::~IcmpSession()
+{ memory::MemoryCap::update_deallocations(sizeof(*this)); }
bool IcmpSession::setup(Packet*)
{
{
public:
IcmpSession(snort::Flow*);
+ ~IcmpSession() override;
bool setup(snort::Packet*) override;
void update_direction(char dir, const snort::SfIp*, uint16_t port) override;
#include "log/messages.h"
#include "main/snort.h"
#include "main/snort_config.h"
+#include "memory/memory_cap.h"
#include "packet_io/active.h"
#include "packet_io/sfdaq.h"
#include "profiler/profiler_defs.h"
{
delete[] fptr;
ip_stats.nodes_released++;
+ memory::MemoryCap::update_deallocations(sizeof(*this) + flen);
}
uint8_t* data = nullptr; /* ptr to adjusted start position */
inline void init(uint16_t flen, const uint8_t* fptr, int ord)
{
assert(flen > 0);
+ memory::MemoryCap::update_allocations(sizeof(*this) + flen);
this->flen = flen;
this->fptr = new uint8_t[flen];
#include "ip_session.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler_defs.h"
#include "protocols/packet.h"
//-------------------------------------------------------------------------
IpSession::IpSession(Flow* flow) : Session(flow)
-{
-}
+{ memory::MemoryCap::update_allocations(sizeof(*this)); }
+
+IpSession::~IpSession()
+{ memory::MemoryCap::update_deallocations(sizeof(*this)); }
void IpSession::clear()
{
{
public:
IpSession(snort::Flow*);
+ ~IpSession() override;
bool setup(snort::Packet*) override;
int process(snort::Packet*) override;
// tcp module stuff
//-------------------------------------------------------------------------
-void TcpStreamSession::sinit()
-{
- //AtomSplitter::init(); // FIXIT-L PAF implement
-}
-
-void TcpStreamSession::sterm()
-{ }
-
void TcpStreamSession::print()
{
char buf[64];
server.print();
}
-
uint16_t get_mss(bool to_server) const;
uint8_t get_tcp_options_len(bool to_server) const;
- static void sinit();
- static void sterm();
-
void reset();
void start_proxy();
void print();
void StreamTcp::tinit()
{
TcpHAManager::tinit();
+ TcpSession::sinit();
}
void StreamTcp::tterm()
{
TcpHAManager::tterm();
+ TcpSession::sterm();
}
NORETURN_ASSERT void StreamTcp::eval(Packet*)
return new TcpSession(lws);
}
-static void tcp_tinit()
-{
- TcpSession::sinit();
-}
-
-static void tcp_tterm()
-{
- TcpSession::sterm();
-}
-
static const InspectApi tcp_api =
{
{
nullptr, // service
nullptr, // init
nullptr, // term
- tcp_tinit,
- tcp_tterm,
+ nullptr, // tinit,
+ nullptr, // tterm,
tcp_ctor,
tcp_dtor,
tcp_ssn,
#include "detection/detection_engine.h"
#include "log/log.h"
#include "main/snort.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler.h"
#include "protocols/packet_manager.h"
#include "time/packet_time.h"
// FIXIT-L don't allocate overlapped part
tsn = TcpSegmentNode::init(tsd);
+
tsn->offset = slide;
tsn->c_len = (uint16_t)newSize;
tsn->i_len = (uint16_t)newSize;
}
break;
}
-
- if ( flushed and !trs.sos.session->flow->two_way_traffic() and !p->ptrs.tcph->is_syn() )
+ // FIXIT-H a drop rule will yoink the seglist out from under us
+ // because apply_delayed_action is only deferred to end of context
+ if ( flushed and trs.sos.seg_count and
+ !trs.sos.session->flow->two_way_traffic() and !p->ptrs.tcph->is_syn() )
{
TcpStreamTracker::TcpState peer = trs.tracker->session->get_peer_state(trs.tracker);
#include "tcp_segment_node.h"
+#include "main/thread.h"
+#include "memory/memory_cap.h"
#include "utils/util.h"
#include "segment_overlap_editor.h"
#include "tcp_module.h"
-TcpSegmentNode::TcpSegmentNode(const struct timeval& tv, const uint8_t* payload, uint16_t len) :
- prev(nullptr), next(nullptr), tv(tv), ts(0), i_seq(0), c_seq(0), i_len(len),
- c_len(len), offset(0)
+#define USE_RESERVE
+#ifdef USE_RESERVE
+static THREAD_LOCAL TcpSegmentNode* reserved = nullptr;
+static THREAD_LOCAL unsigned reserve_sz = 0;
+
+static constexpr unsigned num_res = 4096;
+static constexpr unsigned res_min = 1024;
+static constexpr unsigned res_max = 1460;
+#endif
+
+void TcpSegmentNode::setup()
+{
+#ifdef USE_RESERVE
+ reserved = nullptr;
+ reserve_sz = 0;
+#endif
+}
+
+void TcpSegmentNode::clear()
{
- data = ( uint8_t* )snort_alloc(len);
- memcpy(data, payload, len);
- tcpStats.mem_in_use += len;
+#ifdef USE_RESERVE
+ while ( reserved )
+ {
+ TcpSegmentNode* tsn = reserved;
+ reserved = reserved->next;
+ memory::MemoryCap::update_deallocations(sizeof(*tsn) + tsn->size);
+ tcpStats.mem_in_use -= tsn->size;
+ snort_free(tsn);
+ }
+ reserve_sz = 0;
+#endif
}
//-------------------------------------------------------------------------
// TcpSegment stuff
//-------------------------------------------------------------------------
+
+TcpSegmentNode* TcpSegmentNode::create(
+ const struct timeval& tv, const uint8_t* payload, uint16_t len)
+{
+ TcpSegmentNode* tsn;
+
+#ifdef USE_RESERVE
+ if ( reserved and len > res_min and len <= res_max )
+ {
+ tsn = reserved;
+ reserved = tsn->next;
+ --reserve_sz;
+ }
+ else
+#endif
+ {
+ size_t size = sizeof(*tsn) + len;
+ memory::MemoryCap::update_allocations(size);
+ tsn = (TcpSegmentNode*)snort_alloc(size);
+ tsn->size = len;
+ tcpStats.mem_in_use += len;
+ }
+ tsn->tv = tv;
+ tsn->i_len = tsn->c_len = len;
+ memcpy(tsn->data, payload, len);
+
+ tsn->prev = tsn->next = nullptr;
+ tsn->i_seq = tsn->c_seq = 0;
+ tsn->offset = 0;
+ tsn->ts = 0;
+
+ return tsn;
+}
+
TcpSegmentNode* TcpSegmentNode::init(TcpSegmentDescriptor& tsd)
{
- return new TcpSegmentNode(tsd.get_pkt()->pkth->ts, tsd.get_pkt()->data, tsd.get_seg_len());
+ return create(tsd.get_pkt()->pkth->ts, tsd.get_pkt()->data, tsd.get_seg_len());
}
TcpSegmentNode* TcpSegmentNode::init(TcpSegmentNode& tns)
{
- return new TcpSegmentNode(tns.tv, tns.payload(), tns.c_len);
+ return create(tns.tv, tns.payload(), tns.c_len);
}
void TcpSegmentNode::term()
{
- snort_free(data);
+#ifdef USE_RESERVE
+ if ( size == res_max and reserve_sz < num_res )
+ {
+ next = reserved;
+ reserved = this;
+ reserve_sz++;
+ }
+ else
+#endif
+ {
+ memory::MemoryCap::update_deallocations(sizeof(*this) + size);
+ tcpStats.mem_in_use -= size;
+ snort_free(this);
+ }
tcpStats.segs_released++;
- tcpStats.mem_in_use -= i_len;
- delete this;
}
-bool TcpSegmentNode::is_retransmit(const uint8_t* rdata, uint16_t rsize, uint32_t rseq, uint16_t orig_dsize, bool *full_retransmit)
+bool TcpSegmentNode::is_retransmit(
+ const uint8_t* rdata, uint16_t rsize, uint32_t rseq, uint16_t orig_dsize,
+ bool *full_retransmit)
{
// retransmit must have same payload at same place
if ( !SEQ_EQ(i_seq, rseq) )
// we make a lot of TcpSegments so it is organized by member
// size/alignment requirements to minimize unused space
// ... however, use of padding below is critical, adjust if needed
+// and we use the struct hack to avoid 2 allocs per node
//-----------------------------------------------------------------
class TcpSegmentNode
{
-public:
- TcpSegmentNode(const struct timeval& tv, const uint8_t* segment, uint16_t len);
+private:
+ static TcpSegmentNode* create(const struct timeval& tv, const uint8_t* segment, uint16_t len);
- static TcpSegmentNode* init(TcpSegmentDescriptor& tsd);
- static TcpSegmentNode* init(TcpSegmentNode& tns);
- static TcpSegmentNode* init(const struct timeval&, const uint8_t*, unsigned);
+public:
+ static TcpSegmentNode* init(TcpSegmentDescriptor&);
+ static TcpSegmentNode* init(TcpSegmentNode&);
void term();
+
+ static void setup();
+ static void clear();
+
bool is_retransmit(const uint8_t*, uint16_t size, uint32_t, uint16_t, bool*);
+
uint8_t* payload()
{ return data + offset; }
return (c_seq + c_len) < to_seq;
}
+public:
TcpSegmentNode* prev;
TcpSegmentNode* next;
- uint8_t* data;
-
struct timeval tv;
uint32_t ts;
uint32_t i_seq; // initial seq # of the data segment
uint16_t i_len; // initial length of the data segment
uint16_t c_len; // length of data remaining for reassembly
uint16_t offset;
+ uint16_t size; // actual allocated size (overlaps cause i_len to differ)
+
+ uint8_t data[1];
};
class TcpSegmentList
#include "detection/detection_engine.h"
#include "detection/rules.h"
#include "log/log.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler.h"
#include "protocols/eth.h"
#include "tcp_module.h"
#include "tcp_normalizers.h"
#include "tcp_reassemblers.h"
+#include "tcp_segment_node.h"
#include "tcp_stream_state_machine.h"
using namespace snort;
+void TcpSession::sinit()
+{ TcpSegmentNode::setup(); }
+
+void TcpSession::sterm()
+{ TcpSegmentNode::clear(); }
+
TcpSession::TcpSession(Flow* flow)
: TcpStreamSession(flow)
{
client.session = this;
server.session = this;
tcpStats.instantiated++;
+
+ memory::MemoryCap::update_allocations(sizeof(*this));
}
TcpSession::~TcpSession()
{
clear_session(true, false, false);
+ memory::MemoryCap::update_deallocations(sizeof(*this));
}
bool TcpSession::setup(Packet* p)
TcpSession(snort::Flow*);
~TcpSession() override;
+ static void sinit();
+ static void sterm();
+
bool setup(snort::Packet*) override;
void restart(snort::Packet* p) override;
void precheck(snort::Packet* p) override;
#include "flow/session.h"
#include "framework/data_bus.h"
#include "hash/xhash.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler_defs.h"
#include "protocols/packet.h"
//-------------------------------------------------------------------------
UdpSession::UdpSession(Flow* flow) : Session(flow)
-{
-}
+{ memory::MemoryCap::update_allocations(sizeof(*this)); }
+
+UdpSession::~UdpSession()
+{ memory::MemoryCap::update_deallocations(sizeof(*this)); }
bool UdpSession::setup(Packet* p)
{
{
UdpSessionCleanup(flow);
UdpHAManager::process_deletion(flow);
- flow->clear();
}
void UdpSession::update_direction(
{
public:
UdpSession(snort::Flow*);
+ ~UdpSession() override;
bool setup(snort::Packet*) override;
void update_direction(char dir, const snort::SfIp*, uint16_t port) override;
#include "detection/detection_engine.h"
#include "detection/rules.h"
#include "main/snort.h"
+#include "memory/memory_cap.h"
#include "profiler/profiler_defs.h"
#include "protocols/packet.h"
#include "utils/util.h"
UserSegment* UserSegment::init(const uint8_t* p, unsigned n)
{
unsigned bucket = (n > BUCKET) ? n : BUCKET;
- UserSegment* us = (UserSegment*)snort_alloc(sizeof(*us)+bucket-1);
+ unsigned size = sizeof(UserSegment) + bucket -1;
+ memory::MemoryCap::update_allocations(size);
+ UserSegment* us = (UserSegment*)snort_alloc(size);
+
+ us->size = size;
us->len = 0;
us->offset = 0;
us->used = 0;
void UserSegment::term(UserSegment* us)
{
+ memory::MemoryCap::update_deallocations(us->size);
snort_free(us);
}
// UserSession methods
//-------------------------------------------------------------------------
-UserSession::UserSession(Flow* flow) : Session(flow) { }
+UserSession::UserSession(Flow* flow) : Session(flow)
+{ memory::MemoryCap::update_allocations(sizeof(*this)); }
+UserSession::~UserSession()
+{ memory::MemoryCap::update_deallocations(sizeof(*this)); }
bool UserSession::setup(Packet*)
{
{
client.term();
server.term();
- flow->restart();
}
void UserSession::set_splitter(bool c2s, StreamSplitter* ss)
unsigned len;
unsigned offset;
unsigned used;
+ unsigned size;
uint8_t data[1];
};
{
public:
UserSession(snort::Flow*);
+ ~UserSession() override;
bool setup(snort::Packet*) override;
void clear() override;
projects / thirdparty / snort3.git / commitdiff
