From: Davis McPherson (davmcphe) <davmcphe@cisco.com>
Date: Sat, 17 Aug 2019 13:11:24 +0000 (-0400)
Subject: Merge pull request #1709 in SNORT/snort3 from ~DDAHIPHA/snort3:reload_adjust_memcaps... 
X-Git-Tag: 3.0.0-259~2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4b753009f1fd604446becd729ccd4c94413462f9;p=thirdparty%2Fsnort3.git

Merge pull request #1709 in SNORT/snort3 from ~DDAHIPHA/snort3:reload_adjust_memcaps to master

Squashed commit of the following:

commit 5dd84ba87aebf1d021a4b1d16fdc866dc932fc69
Author: Devendra Dahiphale <ddahipha@cisco.com>
Date:   Tue Aug 13 15:13:36 2019 -0400

    main: Implement reload memcap framework
---

diff --git a/src/framework/module.h b/src/framework/module.h
index 61e883e1d..5b0838b11 100644
--- a/src/framework/module.h
+++ b/src/framework/module.h
@@ -75,6 +75,17 @@ struct RuleMap
     const char* msg;
 };
 
+class ReloadMemcapManager
+{
+public:
+    virtual ~ReloadMemcapManager() = default;
+
+    virtual bool tune_memcap() = 0;
+    virtual bool tune_memcap_idle() = 0;
+protected:
+    ReloadMemcapManager() = default;
+};
+
 class SO_PUBLIC Module
 {
 public:
@@ -185,6 +196,9 @@ public:
 
     void enable_trace();
 
+    const ReloadMemcapManager* get_reload_mcm() const
+    { return reload_mcm; }
+
 protected:
     Module(const char* name, const char* help);
     Module(const char* name, const char* help, const Parameter*,
@@ -206,6 +220,7 @@ private:
     int table_level = 0;
 
     Trace* trace;
+    ReloadMemcapManager* reload_mcm = nullptr;
 
     void set_peg_count(int index, PegCount value)
     {
diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc
index 2891df194..fe58a0ff7 100644
--- a/src/main/analyzer.cc
+++ b/src/main/analyzer.cc
@@ -75,6 +75,7 @@ using namespace std;
 static MainHook_f main_hook = snort_ignore;
 
 THREAD_LOCAL ProfileStats daqPerfStats;
+THREAD_LOCAL std::list<snort::ReloadMemcapManager *> *rel_managers;
 
 static THREAD_LOCAL Analyzer* local_analyzer = nullptr;
 
@@ -526,6 +527,7 @@ void Analyzer::reinit(SnortConfig* sc)
 {
     InspectorManager::thread_reinit(sc);
     ActionManager::thread_reinit(sc);
+    rel_managers = new std::list<snort::ReloadMemcapManager *>(sc->get_reload_memcap_managers());
 }
 
 void Analyzer::term()
@@ -579,6 +581,8 @@ void Analyzer::term()
     Active::thread_term();
     delete switcher;
 
+    delete rel_managers;
+
     sfthreshold_free();
     RateFilter_Cleanup();
 }
@@ -665,13 +669,22 @@ bool Analyzer::handle_command()
 
     ac->execute(*this);
 
-    completed_work_queue_mutex.lock();
-    completed_work_queue.push(ac);
-    completed_work_queue_mutex.unlock();
+    add_command_to_completed_queue(ac);
 
     return true;
 }
 
+void Analyzer::add_command_to_completed_queue(AnalyzerCommand *ac)
+{
+    if (ac->is_complete())
+    {
+        completed_work_queue_mutex.lock();
+        completed_work_queue.push(ac);
+        completed_work_queue_mutex.unlock();
+    } else
+        cache_analyzer_command(ac);
+}
+
 void Analyzer::handle_commands()
 {
     while (handle_command())
@@ -713,6 +726,21 @@ DAQ_RecvStatus Analyzer::process_messages()
         process_daq_msg(msg, false);
         DetectionEngine::onload();
         process_retry_queue();
+
+        if (rel_managers and rel_managers->size())
+        {
+            auto manager = rel_managers->front();
+            if (manager->tune_memcap())
+            {
+                rel_managers->pop_front();
+            }
+        }
+        else
+        {
+            if(ac)
+                add_command_to_completed_queue(ac);
+        }
+
     }
 
     if (exit_after_cnt && (exit_after_cnt -= num_recv) == 0)
diff --git a/src/main/analyzer.h b/src/main/analyzer.h
index 0d4bfc826..55ebb66db 100644
--- a/src/main/analyzer.h
+++ b/src/main/analyzer.h
@@ -30,6 +30,7 @@
 #include <mutex>
 #include <queue>
 #include <string>
+#include <list>
 
 #include "thread.h"
 
@@ -45,6 +46,7 @@ class SFDAQInstance;
 struct Packet;
 struct SnortConfig;
 struct ProfileStats;
+class ReloadMemcapManager;
 }
 
 typedef bool (* MainHook_f)(snort::Packet*);
@@ -110,7 +112,9 @@ private:
     void init_unprivileged();
     void term();
     void show_source();
-
+    void cache_analyzer_command(AnalyzerCommand* aci) { ac = aci; }
+    void add_command_to_completed_queue(AnalyzerCommand *ac);
+    AnalyzerCommand* get_analyzer_command() { return ac; }
 public:
     std::queue<AnalyzerCommand*> completed_work_queue;
     std::mutex completed_work_queue_mutex;
@@ -131,6 +135,7 @@ private:
     RetryQueue* retry_queue = nullptr;
     OopsHandler* oops_handler = nullptr;
     ContextSwitcher* switcher = nullptr;
+    AnalyzerCommand* ac = nullptr;
 
     std::mutex pending_work_queue_mutex;
 };
diff --git a/src/main/analyzer_command.h b/src/main/analyzer_command.h
index ea91dd08b..d2ce19d8a 100644
--- a/src/main/analyzer_command.h
+++ b/src/main/analyzer_command.h
@@ -34,8 +34,11 @@ public:
     virtual const char* stringify() = 0;
     unsigned get() { return ++ref_count; }
     unsigned put() { return --ref_count; }
+    bool is_complete() { return completion_status; }
+    void set_completion_status(bool status) { completion_status = status; }
 private:
     unsigned ref_count = 0;
+    bool completion_status = true;
 };
 
 class ACGetStats : public AnalyzerCommand
diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index d4ba8f0b2..170b48e6b 100644
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -310,6 +310,8 @@ SnortConfig::~SnortConfig()
     delete daq_config;
     delete proto_ref;
 
+    reload_managers.clear();
+
     trim_heap();
 }
 
@@ -1084,3 +1086,13 @@ void SnortConfig::set_conf(SnortConfig* sc)
     }
 }
 
+SO_PUBLIC bool SnortConfig::register_reload_memcap_manager(ReloadMemcapManager *memcap_manager)
+{
+    reload_managers.push_back(memcap_manager);
+    return true;
+}
+
+std::list<ReloadMemcapManager *> SnortConfig::get_reload_memcap_managers()
+{
+    return reload_managers;
+}
diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index f66584968..580c7ce9e 100644
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -31,6 +31,8 @@
 #include "main/thread.h"
 #include "sfip/sf_cidr.h"
 
+#include <list>
+
 #define DEFAULT_LOG_DIR "."
 
 enum RunFlag
@@ -148,12 +150,16 @@ class ProtocolReference;
 struct GHash;
 struct XHash;
 
+class ReloadMemcapManager;
+
 struct SnortConfig;
 typedef void (* ScScratchFunc)(SnortConfig* sc);
 
 struct SnortConfig
 {
 private:
+    std::list<ReloadMemcapManager *> reload_managers;
+
     void init(const SnortConfig* const, ProtocolReference*);
     bool verify_stream_inspectors();
 
@@ -164,6 +170,9 @@ public:
 
     SnortConfig(const SnortConfig&) = delete;
 
+    SO_PUBLIC bool register_reload_memcap_manager(ReloadMemcapManager *);
+    std::list<ReloadMemcapManager *> get_reload_memcap_managers();
+
     void setup();
     void post_setup();
     bool verify();
diff --git a/src/stream/CMakeLists.txt b/src/stream/CMakeLists.txt
index 19eb7f09e..62c2dbb0f 100644
--- a/src/stream/CMakeLists.txt
+++ b/src/stream/CMakeLists.txt
@@ -10,6 +10,7 @@ add_subdirectory(file)
 add_subdirectory(test)
 
 set (STREAM_INCLUDES
+    flush_bucket.h
     paf.h
     stream.h
     stream_splitter.h