Squashed commit of the following:

commit 929661c23d43af57f00a98a9df5046960187d526
Author: Russ Combs <rucombs@cisco.com>
Date:   Sat Sep 9 10:04:58 2017 -0400

    build: fix noreturn and unused warnings

commit 03230ffb0c7b45800f8368a4009dbb5b82b34671
Author: Russ Combs <rucombs@cisco.com>
Date:   Sat Sep 9 15:29:47 2017 -0400

    memory: patch around allocation tracking issue

commit 9436ba425e2fa1669ef35046d4a1337b33068652
Author: Russ Combs <rucombs@cisco.com>
Date:   Sat Sep 9 10:03:27 2017 -0400

    memory: remove canary from production builds to reduce overhead

commit 7fadd3d35b6c19fb42e3809db384db4828497f7e
Author: Russ Combs <rucombs@cisco.com>
Date:   Mon Sep 4 18:28:25 2017 -0400

    memory: output basic startup heap stats

diff --git a/src/log/messages.cc b/src/log/messages.cc
index 76e4d5c778af6ca232373410979fce7763149e15..249049387e1c3769e2ff9b5bc6e8da0aad0a7344 100644 (file)
--- a/src/log/messages.cc
+++ b/src/log/messages.cc
@@ -303,7 +303,7 @@ void ErrorMessage(const char* format,...)
     }
 }
 
-[[noreturn]] void log_safec_error(const char* msg, void*, int e)
+void log_safec_error(const char* msg, void*, int e)
 {
     static THREAD_LOCAL unsigned safec_errors = 0;
 

diff --git a/src/main.cc b/src/main.cc
index 1c6df6f0b903db722a589f9b75f9cd753ccb1ec6..6886f409e43839c13a9a1bac304d8712969b4b16 100644 (file)
--- a/src/main.cc
+++ b/src/main.cc
@@ -44,7 +44,6 @@
 #include "managers/inspector_manager.h"
 #include "managers/module_manager.h"
 #include "managers/plugin_manager.h"
-#include "memory/memory_cap.h"
 #include "packet_io/sfdaq.h"
 #include "packet_io/trough.h"
 #include "target_based/sftarget_reader.h"
@@ -842,10 +841,6 @@ static void snort_main()
         pig.set_index(idx);
     }
 
-    memory::MemoryCap::calculate(max_pigs);
-    if ( SnortConfig::log_verbose() )
-        memory::MemoryCap::print();
-
     main_loop();
 
     delete pig_poke;

diff --git a/src/main/snort.cc b/src/main/snort.cc
index e849b40940b97cfbf9d387bb052bf23abab1b095..7de1529c940207a0be7d7efff0282ac238d0263c 100644 (file)
--- a/src/main/snort.cc
+++ b/src/main/snort.cc
@@ -58,6 +58,7 @@
 #include "loggers/loggers.h"
 #include "main.h"
 #include "main/shell.h"
+#include "main/thread_config.h"
 #include "managers/action_manager.h"
 #include "managers/codec_manager.h"
 #include "managers/inspector_manager.h"
@@ -67,6 +68,7 @@
 #include "managers/mpse_manager.h"
 #include "managers/plugin_manager.h"
 #include "managers/script_manager.h"
+#include "memory/memory_cap.h"
 #include "network_inspectors/network_inspectors.h"
 #include "packet_io/active.h"
 #include "packet_io/sfdaq.h"
@@ -542,6 +544,9 @@ void Snort::setup(int argc, char* argv[])
     keep_kmap_lib();
     keep_utf_lib();
 
+    memory::MemoryCap::calculate(ThreadConfig::get_instance_max());
+    memory::MemoryCap::print();
+
     TimeStart();
 }
 

diff --git a/src/memory/memory_cap.cc b/src/memory/memory_cap.cc
index 473fe5aa15c34cbc98e2b56e9ca561bff81f204d..30a543b9146fe64d8b6b0844a1307212e971ba25 100644 (file)
--- a/src/memory/memory_cap.cc
+++ b/src/memory/memory_cap.cc
@@ -29,6 +29,7 @@
 #include "log/messages.h"
 #include "main/snort_config.h"
 #include "profiler/memory_profiler_active_context.h"
+#include "utils/stats.h"
 
 #include "memory_config.h"
 #include "memory_module.h"
@@ -49,15 +50,24 @@ struct Tracker
     size_t allocated = 0;
     size_t deallocated = 0;
 
+    uint64_t allocations = 0;
+    uint64_t deallocations = 0;
+
     void allocate(size_t n)
-    { allocated += n; }
+    { allocated += n; ++allocations; }
 
     void deallocate(size_t n)
-    { deallocated += n; }
+    { deallocated += n; ++deallocations; }
 
     size_t used() const
     {
-        assert(allocated >= deallocated);
+        // FIXIT-H this assertion fails at analyzer.cc:93 / starting packet thread
+        // {allocated = 0, deallocated = 48, allocations = 0, deallocations = 1}
+        //assert(allocated >= deallocated);
+
+        if ( allocated < deallocated )
+            return 0;
+
         return allocated - deallocated;
     }
 
@@ -160,9 +170,10 @@ bool MemoryCap::over_threshold()
 
 void MemoryCap::calculate(unsigned num_threads)
 {
+    assert(!is_packet_thread());
     const MemoryConfig& config = *snort_conf->memory;
 
-    assert(!is_packet_thread());
+    auto main_thread_used = s_tracker.used();
 
     if ( !config.cap )
     {
@@ -170,42 +181,50 @@ void MemoryCap::calculate(unsigned num_threads)
         return;
     }
 
-    auto main_thread_used = s_tracker.used();
-
     if ( main_thread_used > config.cap )
-        FatalError("main thread memory usage (%zu) is greater than cap\n", main_thread_used);
+    {
+        ParseError("main thread memory usage (%zu) is greater than cap\n", main_thread_used);
+        return;
+    }
 
     auto real_cap = config.cap - main_thread_used;
-
     thread_cap = real_cap / num_threads;
 
-    // FIXIT-M we probably want to add some fixed overhead to allow the packet threads to
-    // startup and preallocate flows and whatnot
-    if ( !thread_cap )
-        FatalError("per-thread memory cap is 0");
+    // FIXIT-L do we want to add some fixed overhead to allow the packet threads to
+    // startup and preallocate flows and whatnot?
 
-    DebugFormat(DEBUG_MEMORY, "per-thread memory cap set to %zu\n", thread_cap);
+    if ( !thread_cap )
+    {
+        ParseError("per-thread memory cap is 0");
+        return;
+    }
 
     if ( config.threshold )
-    {
         preemptive_threshold = memory::calculate_threshold(thread_cap, config.threshold);
-        DebugFormat(DEBUG_MEMORY,
-            "per-thread preemptive action threshold set to %zu\n", preemptive_threshold);
-    }
 }
 
 void MemoryCap::print()
 {
     const MemoryConfig& config = *snort_conf->memory;
 
-    LogMessage("memory configuration\n");
-    LogMessage("    global cap: %zu\n", config.cap);
-    LogMessage("    global preemptive threshold percent: %zu\n", config.threshold);
-    LogMessage("    cap type: %s\n", config.soft? "soft" : "hard");
-    LogMessage("    thread cap: %zu\n", thread_cap);
-    LogMessage("    preemptive threshold: %zu\n", preemptive_threshold);
-    LogMessage("    main thread usage: %zu\n", s_tracker.used());
-    LogMessage("\n");
+    if ( SnortConfig::log_verbose() or s_tracker.allocations )
+        LogLabel("memory (heap)");
+
+    if ( SnortConfig::log_verbose() )
+    {
+        LogMessage("    global cap: %zu\n", config.cap);
+        LogMessage("    global preemptive threshold percent: %zu\n", config.threshold);
+        LogMessage("    cap type: %s\n", config.soft? "soft" : "hard");
+    }
+
+    if ( s_tracker.allocations )
+    {
+        LogMessage("    main thread usage: %zu\n", s_tracker.used());
+        LogMessage("    allocations: %zu\n", s_tracker.allocations);
+        LogMessage("    deallocations: %zu\n", s_tracker.deallocations);
+        LogMessage("    thread cap: %zu\n", thread_cap);
+        LogMessage("    preemptive threshold: %zu\n", preemptive_threshold);
+    }
 }
 
 } // namespace memory

diff --git a/src/memory/memory_manager.cc b/src/memory/memory_manager.cc
index 750a8aac8770ccdf7d8e1bd887d9059bdaa69e31..c827176638df34f2e5dc68fa4faeccc23188abf1 100644 (file)
--- a/src/memory/memory_manager.cc
+++ b/src/memory/memory_manager.cc
@@ -43,14 +43,22 @@ namespace memory
 
 struct Metadata
 {
+#if defined(REG_TEST) || defined(UNIT_TEST)
+    static constexpr size_t SANITY_CHECK_VALUE = 0xabcdef;
     size_t sanity;
+#endif
+
     // number of requested bytes
     size_t payload_size;
 
     // total number of bytes allocated, including Metadata header
     size_t total_size() const;
     void* payload_offset();
-    bool valid() const;
+
+#if defined(REG_TEST) || defined(UNIT_TEST)
+    bool valid() const
+    { return sanity == SANITY_CHECK_VALUE; }
+#endif
 
     Metadata(size_t = 0);
 
@@ -60,8 +68,6 @@ struct Metadata
     static Metadata* create(size_t);
 
     static Metadata* extract(void*);
-
-    static size_t SANITY_CHECK_VALUE;
 };
 
 inline size_t Metadata::total_size() const
@@ -70,11 +76,11 @@ inline size_t Metadata::total_size() const
 inline void* Metadata::payload_offset()
 { return this + 1; }
 
-inline bool Metadata::valid() const
-{ return sanity == SANITY_CHECK_VALUE; }
-
 inline Metadata::Metadata(size_t n) :
-    sanity(SANITY_CHECK_VALUE), payload_size(n)
+#if defined(REG_TEST) || defined(UNIT_TEST)
+    sanity(SANITY_CHECK_VALUE),
+#endif
+    payload_size(n)
 { }
 
 inline size_t Metadata::calculate_total_size(size_t n)
@@ -91,7 +97,10 @@ Metadata* Metadata::create(size_t n)
 
     // Trigger metadata ctor
     *meta = Metadata(n);
+
+#if defined(REG_TEST) || defined(UNIT_TEST)
     assert(meta->valid());
+#endif
 
     return meta;
 }
@@ -102,13 +111,13 @@ Metadata* Metadata::extract(void* p)
 
     auto meta = static_cast<Metadata*>(p) - 1;
 
+#if defined(REG_TEST) || defined(UNIT_TEST)
     assert(meta->valid());
+#endif
 
     return meta;
 }
 
-size_t Metadata::SANITY_CHECK_VALUE = 0xabcdef;
-
 // -----------------------------------------------------------------------------
 // the meat
 // -----------------------------------------------------------------------------

diff --git a/src/service_inspectors/sip/ips_sip_method.cc b/src/service_inspectors/sip/ips_sip_method.cc
index 8664e815149d7377dc47cbdeabb6ff9ae34bfd15..64d2e3bd2920f353a8f971d27fdd50cb2c501dd2 100644 (file)
--- a/src/service_inspectors/sip/ips_sip_method.cc
+++ b/src/service_inspectors/sip/ips_sip_method.cc
@@ -179,7 +179,6 @@ bool SipMethodModule::set(const char*, Value& v, SnortConfig*)
     if ( v.is("*method") )
     {
         char* tok = (char*)v.get_string();
-        SIPMethodNode *method = NULL;
 
         if (tok[0] == '!')
         {