From: Russ Combs Date: Sun, 10 Sep 2017 02:36:17 +0000 (-0400) Subject: Squashed commit of the following: X-Git-Tag: 3.0.0-240~48 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=87cb71eaaec55a2cd767e266ec2bff00bff99d47;p=thirdparty%2Fsnort3.git Squashed commit of the following: commit 929661c23d43af57f00a98a9df5046960187d526 Author: Russ Combs Date: Sat Sep 9 10:04:58 2017 -0400 build: fix noreturn and unused warnings commit 03230ffb0c7b45800f8368a4009dbb5b82b34671 Author: Russ Combs Date: Sat Sep 9 15:29:47 2017 -0400 memory: patch around allocation tracking issue commit 9436ba425e2fa1669ef35046d4a1337b33068652 Author: Russ Combs Date: Sat Sep 9 10:03:27 2017 -0400 memory: remove canary from production builds to reduce overhead commit 7fadd3d35b6c19fb42e3809db384db4828497f7e Author: Russ Combs Date: Mon Sep 4 18:28:25 2017 -0400 memory: output basic startup heap stats --- diff --git a/src/log/messages.cc b/src/log/messages.cc index 76e4d5c77..249049387 100644 --- a/src/log/messages.cc +++ b/src/log/messages.cc @@ -303,7 +303,7 @@ void ErrorMessage(const char* format,...) } } -[[noreturn]] void log_safec_error(const char* msg, void*, int e) +void log_safec_error(const char* msg, void*, int e) { static THREAD_LOCAL unsigned safec_errors = 0; diff --git a/src/main.cc b/src/main.cc index 1c6df6f0b..6886f409e 100644 --- a/src/main.cc +++ b/src/main.cc @@ -44,7 +44,6 @@ #include "managers/inspector_manager.h" #include "managers/module_manager.h" #include "managers/plugin_manager.h" -#include "memory/memory_cap.h" #include "packet_io/sfdaq.h" #include "packet_io/trough.h" #include "target_based/sftarget_reader.h" @@ -842,10 +841,6 @@ static void snort_main() pig.set_index(idx); } - memory::MemoryCap::calculate(max_pigs); - if ( SnortConfig::log_verbose() ) - memory::MemoryCap::print(); - main_loop(); delete pig_poke; diff --git a/src/main/snort.cc b/src/main/snort.cc index e849b4094..7de1529c9 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -58,6 +58,7 @@ #include "loggers/loggers.h" #include "main.h" #include "main/shell.h" +#include "main/thread_config.h" #include "managers/action_manager.h" #include "managers/codec_manager.h" #include "managers/inspector_manager.h" @@ -67,6 +68,7 @@ #include "managers/mpse_manager.h" #include "managers/plugin_manager.h" #include "managers/script_manager.h" +#include "memory/memory_cap.h" #include "network_inspectors/network_inspectors.h" #include "packet_io/active.h" #include "packet_io/sfdaq.h" @@ -542,6 +544,9 @@ void Snort::setup(int argc, char* argv[]) keep_kmap_lib(); keep_utf_lib(); + memory::MemoryCap::calculate(ThreadConfig::get_instance_max()); + memory::MemoryCap::print(); + TimeStart(); } diff --git a/src/memory/memory_cap.cc b/src/memory/memory_cap.cc index 473fe5aa1..30a543b91 100644 --- a/src/memory/memory_cap.cc +++ b/src/memory/memory_cap.cc @@ -29,6 +29,7 @@ #include "log/messages.h" #include "main/snort_config.h" #include "profiler/memory_profiler_active_context.h" +#include "utils/stats.h" #include "memory_config.h" #include "memory_module.h" @@ -49,15 +50,24 @@ struct Tracker size_t allocated = 0; size_t deallocated = 0; + uint64_t allocations = 0; + uint64_t deallocations = 0; + void allocate(size_t n) - { allocated += n; } + { allocated += n; ++allocations; } void deallocate(size_t n) - { deallocated += n; } + { deallocated += n; ++deallocations; } size_t used() const { - assert(allocated >= deallocated); + // FIXIT-H this assertion fails at analyzer.cc:93 / starting packet thread + // {allocated = 0, deallocated = 48, allocations = 0, deallocations = 1} + //assert(allocated >= deallocated); + + if ( allocated < deallocated ) + return 0; + return allocated - deallocated; } @@ -160,9 +170,10 @@ bool MemoryCap::over_threshold() void MemoryCap::calculate(unsigned num_threads) { + assert(!is_packet_thread()); const MemoryConfig& config = *snort_conf->memory; - assert(!is_packet_thread()); + auto main_thread_used = s_tracker.used(); if ( !config.cap ) { @@ -170,42 +181,50 @@ void MemoryCap::calculate(unsigned num_threads) return; } - auto main_thread_used = s_tracker.used(); - if ( main_thread_used > config.cap ) - FatalError("main thread memory usage (%zu) is greater than cap\n", main_thread_used); + { + ParseError("main thread memory usage (%zu) is greater than cap\n", main_thread_used); + return; + } auto real_cap = config.cap - main_thread_used; - thread_cap = real_cap / num_threads; - // FIXIT-M we probably want to add some fixed overhead to allow the packet threads to - // startup and preallocate flows and whatnot - if ( !thread_cap ) - FatalError("per-thread memory cap is 0"); + // FIXIT-L do we want to add some fixed overhead to allow the packet threads to + // startup and preallocate flows and whatnot? - DebugFormat(DEBUG_MEMORY, "per-thread memory cap set to %zu\n", thread_cap); + if ( !thread_cap ) + { + ParseError("per-thread memory cap is 0"); + return; + } if ( config.threshold ) - { preemptive_threshold = memory::calculate_threshold(thread_cap, config.threshold); - DebugFormat(DEBUG_MEMORY, - "per-thread preemptive action threshold set to %zu\n", preemptive_threshold); - } } void MemoryCap::print() { const MemoryConfig& config = *snort_conf->memory; - LogMessage("memory configuration\n"); - LogMessage(" global cap: %zu\n", config.cap); - LogMessage(" global preemptive threshold percent: %zu\n", config.threshold); - LogMessage(" cap type: %s\n", config.soft? "soft" : "hard"); - LogMessage(" thread cap: %zu\n", thread_cap); - LogMessage(" preemptive threshold: %zu\n", preemptive_threshold); - LogMessage(" main thread usage: %zu\n", s_tracker.used()); - LogMessage("\n"); + if ( SnortConfig::log_verbose() or s_tracker.allocations ) + LogLabel("memory (heap)"); + + if ( SnortConfig::log_verbose() ) + { + LogMessage(" global cap: %zu\n", config.cap); + LogMessage(" global preemptive threshold percent: %zu\n", config.threshold); + LogMessage(" cap type: %s\n", config.soft? "soft" : "hard"); + } + + if ( s_tracker.allocations ) + { + LogMessage(" main thread usage: %zu\n", s_tracker.used()); + LogMessage(" allocations: %zu\n", s_tracker.allocations); + LogMessage(" deallocations: %zu\n", s_tracker.deallocations); + LogMessage(" thread cap: %zu\n", thread_cap); + LogMessage(" preemptive threshold: %zu\n", preemptive_threshold); + } } } // namespace memory diff --git a/src/memory/memory_manager.cc b/src/memory/memory_manager.cc index 750a8aac8..c82717663 100644 --- a/src/memory/memory_manager.cc +++ b/src/memory/memory_manager.cc @@ -43,14 +43,22 @@ namespace memory struct Metadata { +#if defined(REG_TEST) || defined(UNIT_TEST) + static constexpr size_t SANITY_CHECK_VALUE = 0xabcdef; size_t sanity; +#endif + // number of requested bytes size_t payload_size; // total number of bytes allocated, including Metadata header size_t total_size() const; void* payload_offset(); - bool valid() const; + +#if defined(REG_TEST) || defined(UNIT_TEST) + bool valid() const + { return sanity == SANITY_CHECK_VALUE; } +#endif Metadata(size_t = 0); @@ -60,8 +68,6 @@ struct Metadata static Metadata* create(size_t); static Metadata* extract(void*); - - static size_t SANITY_CHECK_VALUE; }; inline size_t Metadata::total_size() const @@ -70,11 +76,11 @@ inline size_t Metadata::total_size() const inline void* Metadata::payload_offset() { return this + 1; } -inline bool Metadata::valid() const -{ return sanity == SANITY_CHECK_VALUE; } - inline Metadata::Metadata(size_t n) : - sanity(SANITY_CHECK_VALUE), payload_size(n) +#if defined(REG_TEST) || defined(UNIT_TEST) + sanity(SANITY_CHECK_VALUE), +#endif + payload_size(n) { } inline size_t Metadata::calculate_total_size(size_t n) @@ -91,7 +97,10 @@ Metadata* Metadata::create(size_t n) // Trigger metadata ctor *meta = Metadata(n); + +#if defined(REG_TEST) || defined(UNIT_TEST) assert(meta->valid()); +#endif return meta; } @@ -102,13 +111,13 @@ Metadata* Metadata::extract(void* p) auto meta = static_cast(p) - 1; +#if defined(REG_TEST) || defined(UNIT_TEST) assert(meta->valid()); +#endif return meta; } -size_t Metadata::SANITY_CHECK_VALUE = 0xabcdef; - // ----------------------------------------------------------------------------- // the meat // ----------------------------------------------------------------------------- diff --git a/src/service_inspectors/sip/ips_sip_method.cc b/src/service_inspectors/sip/ips_sip_method.cc index 8664e8151..64d2e3bd2 100644 --- a/src/service_inspectors/sip/ips_sip_method.cc +++ b/src/service_inspectors/sip/ips_sip_method.cc @@ -179,7 +179,6 @@ bool SipMethodModule::set(const char*, Value& v, SnortConfig*) if ( v.is("*method") ) { char* tok = (char*)v.get_string(); - SIPMethodNode *method = NULL; if (tok[0] == '!') {