}
/*!
- * Try to change the memcap
+ * Try to change the memcap
* Behavior is undefined when t->usrfree is set
*
* t SFXHASH table pointer
return XHASH_OK;
}
-
/*!
* Delete the hash Table
*
- * free key's, free node's, and free the users data.
+ * free keys, free nodes, and free the users data.
*
* h XHash table pointer
*
}
/*
- * Allocat a new hash node, uses Auto Node Recovery if needed and enabled.
+ * Allocate a new hash node, uses Auto Node Recovery if needed and enabled.
*
* The oldest node is the one with the longest time since it was last touched,
* and does not have any direct indication of how long the node has been around.
* t XHash table pointer
* key users key pointer
*
- * return integer
- * retval XHASH_OK success
- * retval XHASH_INTABLE already in the table, t->cnode points to the node
- * retval XHASH_NOMEM not enough memory
+ * return XHashNode*
+ * retval nullptr failed to add node
*/
XHashNode* xhash_get_node(XHash* t, const void* key)
{
return hnode;
}
+/*!
+ * Add a key to the hash table, return the hash node
+ * If space is not available, it will remove a single
+ * node (the LRU) and replace it with a new node.
+ *
+ * This function handles the scenario that we don't
+ * have enough room to allocate a new node - if this is the
+ * case, additional pruning should be done
+ *
+ * t XHash table pointer
+ * key users key pointer
+ * prune_performed bool pointer, 1 = successful prune, 0 = no/failed prune
+ *
+ * return XHashNode*
+ * retval XhashNode* successfully allocated
+ * retval nullptr failed to allocate
+ */
+XHashNode* xhash_get_node_with_prune(XHash* t, const void* key, bool* prune_performed)
+{
+ size_t mem_after_alloc = t->mc.memused + xhash_required_mem(t);
+ bool over_capacity = (t->mc.memcap < mem_after_alloc);
+ XHashNode* hnode = nullptr;
+
+ if (over_capacity)
+ *prune_performed = (xhash_free_anr_lru(t) == XHASH_OK);
+
+ if (*prune_performed or !over_capacity)
+ hnode = xhash_get_node(t, key);
+
+ return hnode;
+}
+
/*!
* Find a Node based on the key
*
return XHASH_ERR;
}
+/*!
+ * Unlink and free nodes if the xhash is exceeding the
+ * specified memcap
+ *
+ * t XHash table pointer
+ * num_freed set to the number of nodes freed
+ * work_limit the max amount of work to do
+ *
+ * returns
+ * XHASH_ERR if error occurs
+ * XHASH_OK if node(s) freed and memused is within memcap
+ * XHASH_PENDING if additional pending is required
+ */
+int xhash_free_overallocations(XHash* t, unsigned work_limit, unsigned* num_freed)
+{
+
+ while (t->mc.memcap < t->mc.memused and work_limit--)
+ {
+ if (xhash_free_anr_lru(t) != XHASH_OK)
+ return XHASH_ERR;
+
+ ++*num_freed;
+ }
+
+ return (t->mc.memcap > t->mc.memused) ? XHASH_OK : XHASH_PENDING;
+}
+
/*!
* Find and return the first hash table node
*
SO_PUBLIC void xhash_set_max_nodes(XHash* h, int max_nodes);
SO_PUBLIC int xhash_change_memcap(XHash *t, unsigned long new_memcap, unsigned *max_work);
+SO_PUBLIC int xhash_free_overallocations(XHash* t, unsigned work_limit, unsigned* num_freed);
SO_PUBLIC void xhash_delete(XHash* h);
SO_PUBLIC int xhash_make_empty(XHash*);
SO_PUBLIC int xhash_add(XHash* h, void* key, void* data);
SO_PUBLIC XHashNode* xhash_get_node(XHash* t, const void* key);
+SO_PUBLIC XHashNode* xhash_get_node_with_prune(XHash* t, const void* key, bool* prune_performed);
SO_PUBLIC int xhash_remove(XHash* h, void* key);
// Get the # of Nodes in HASH the table
inline unsigned xhash_overhead_blocks(XHash* t)
{ return t->overhead_blocks; }
+// Get the amount of space required to allocate a new node in the xhash t.
+inline size_t xhash_required_mem(XHash *t)
+{ return sizeof(XHashNode) + t->pad + t->keysize + t->datasize; }
+
SO_PUBLIC int xhash_free_anr_lru(XHash* t);
SO_PUBLIC void* xhash_mru(XHash* t);
SO_PUBLIC void* xhash_lru(XHash* t);
static void cleanup();
static bool is_starting();
- static bool is_reloading();
static bool has_dropped_privileges();
+ SO_PUBLIC static bool is_reloading();
private:
static void init(int, char**);
perf_module.cc
perf_module.h
perf_monitor.cc
+ perf_pegs.h
+ perf_reload_tuner.h
perf_tracker.cc
perf_tracker.h
text_formatter.cc
#include "log/messages.h"
#include "protocols/packet.h"
+#include "perf_pegs.h"
+
using namespace snort;
+// The default number of rows used for the xhash ip_map
+#define DEFAULT_XHASH_NROWS 1021
#define TRACKER_NAME PERF_NAME "_flow_ip"
struct FlowStateKey
{
FlowStateKey key;
FlowStateValue* value = nullptr;
+ bool prune_required = false;
if (src_addr->less_than(*dst_addr))
{
value = (FlowStateValue*)xhash_find(ip_map, &key);
if (!value)
{
- XHashNode* node = xhash_get_node(ip_map, &key);
+ XHashNode* node = xhash_get_node_with_prune(ip_map, &key, &prune_required);
if (!node)
{
return nullptr;
}
+
+ if (prune_required)
+ {
+ ++pmstats.total_frees;
+ ++pmstats.alloc_prunes;
+ }
+
memset(node->data, 0, sizeof(FlowStateValue));
value = (FlowStateValue*)node->data;
}
return value;
}
+bool FlowIPTracker::initialize(size_t new_memcap)
+{
+ bool need_pruning = false;
+
+ if (!ip_map)
+ {
+ ip_map = xhash_new(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue),
+ new_memcap, 1, nullptr, nullptr, 1);
+ }
+ else
+ {
+ need_pruning = (new_memcap < memcap);
+ memcap = new_memcap;
+ ip_map->mc.memcap = memcap;
+ }
+
+ return need_pruning;
+}
+
FlowIPTracker::FlowIPTracker(PerfConfig* perf) : PerfTracker(perf, TRACKER_NAME),
- perf_flags(perf->perf_flags)
+ perf_flags(perf->perf_flags), perf_conf(perf)
{
formatter->register_section("flow_ip");
formatter->register_field("ip_a", ip_a);
&stats.state_changes[SFS_STATE_UDP_CREATED]);
formatter->finalize_fields();
- ip_map = xhash_new(1021, sizeof(FlowStateKey), sizeof(FlowStateValue),
- perf->flowip_memcap, 1, nullptr, nullptr, 1);
+ memcap = perf->flowip_memcap;
+
+ ip_map = xhash_new(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue),
+ memcap, 1, nullptr, nullptr, 1);
if (!ip_map)
FatalError("Unable to allocate memory for FlowIP stats\n");
#ifndef FLOW_IP_TRACKER_H
#define FLOW_IP_TRACKER_H
-#include "perf_tracker.h"
-
#include "hash/xhash.h"
+#include "perf_tracker.h"
+
enum FlowState
{
SFS_STATE_TCP_ESTABLISHED = 0,
FlowIPTracker(PerfConfig* perf);
~FlowIPTracker() override;
+ bool initialize(size_t new_memcap);
void reset() override;
void update(snort::Packet*) override;
void process(bool) override;
-
int update_state(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, FlowState);
+ snort::XHash* get_ip_map()
+ { return ip_map; }
private:
FlowStateValue stats;
snort::XHash* ip_map;
char ip_a[41], ip_b[41];
int perf_flags;
-
+ PerfConfig* perf_conf;
+ size_t memcap;
FlowStateValue* find_stats(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, int* swapped);
void write_stats();
void display_stats();
+
};
#endif
#include "perf_module.h"
#include "log/messages.h"
+#include "main/snort.h"
#include "managers/module_manager.h"
+#include "perf_pegs.h"
+#include "perf_reload_tuner.h"
+
using namespace snort;
//-------------------------------------------------------------------------
return true;
}
-bool PerfMonModule::end(const char* fqn, int idx, SnortConfig*)
+bool PerfMonModule::end(const char* fqn, int idx, SnortConfig* sc)
{
+
+ perfmon_rrt.set_memcap(config->flowip_memcap);
+
+ if ( Snort::is_reloading() )
+ {
+ sc->register_reload_resource_tuner(perfmon_rrt);
+ }
+
if ( idx != 0 && strcmp(fqn, "perf_monitor.modules") == 0 )
return config->modules.back().confirm_parse();
}
PerfConfig* PerfMonModule::get_config()
-{
+{
PerfConfig* tmp = config;
config = nullptr;
- return tmp;
+ return tmp;
}
const PegInfo* PerfMonModule::get_pegs() const
-{ return simple_pegs; }
+{ return perf_module_pegs; }
PegCount* PerfMonModule::get_counts() const
{ return (PegCount*)&pmstats; }
{
if ( modules.empty() )
{
- auto all_modules = ModuleManager::get_all_modules();
+ auto all_modules = ModuleManager::get_all_modules();
for ( auto& mod : all_modules )
{
ModuleConfig cfg;
#include "framework/module.h"
+#include "perf_pegs.h"
+#include "perf_reload_tuner.h"
+
#define PERF_NAME "perf_monitor"
#define PERF_HELP "performance monitoring and flow statistics collection"
private:
PerfConfig* config = nullptr;
+ PerfMonReloadTuner perfmon_rrt;
};
-extern THREAD_LOCAL SimpleStats pmstats;
+extern THREAD_LOCAL PerfPegStats pmstats;
extern THREAD_LOCAL snort::ProfileStats perfmonStats;
#endif
#endif
#include "framework/data_bus.h"
+#include "hash/xhash.h"
#include "log/messages.h"
#include "managers/inspector_manager.h"
#include "profiler/profiler.h"
#include "flow_tracker.h"
#include "perf_module.h"
+
#ifdef UNIT_TEST
#include "catch/snort_catch.h"
#endif
using namespace snort;
-THREAD_LOCAL SimpleStats pmstats;
+THREAD_LOCAL PerfPegStats pmstats;
THREAD_LOCAL ProfileStats perfmonStats;
static THREAD_LOCAL std::vector<PerfTracker*>* trackers;
static THREAD_LOCAL FlowIPTracker* flow_ip_tracker = nullptr;
+
//-------------------------------------------------------------------------
// class stuff
//-------------------------------------------------------------------------
private:
PerfConfig* const config;
-
void disable_tracker(size_t);
};
tracker->reset();
}
+bool PerfMonReloadTuner::tinit()
+{
+ if (flow_ip_tracker)
+ return flow_ip_tracker->initialize(memcap);
+ else
+ return false;
+}
+
+bool PerfMonReloadTuner::tune_resources(unsigned work_limit)
+{
+ if (flow_ip_tracker)
+ {
+ unsigned num_freed = 0;
+ int result = xhash_free_overallocations(flow_ip_tracker->get_ip_map(), work_limit, &num_freed);
+ pmstats.total_frees += num_freed;
+ pmstats.reload_frees += num_freed;
+ return (result == XHASH_OK);
+ }
+ else
+ {
+ return false;
+ }
+}
+
void PerfMonitor::tterm()
{
if (trackers)
#ifdef UNIT_TEST
TEST_CASE("Process timing logic", "[perfmon]")
{
- PerfMonModule mod;
+ PerfMonModule mod;
PerfConfig* config = new PerfConfig;
mod.set_config(config);
PerfMonitor perfmon(mod.get_config());
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// perf_pegs.h author Michael Matirko <mmatirko@cisco.com>
+
+
+#ifndef PERF_PEGS_H
+#define PERF_PEGS_H
+
+#include "framework/counts.h"
+#include "main/snort_types.h"
+
+static const PegInfo perf_module_pegs[] =
+{
+
+ { CountType::SUM, "packets", "total packets processed by performance monitor" },
+ { CountType::SUM, "total_frees", "total flows pruned or freed by performance monitor" },
+ { CountType::SUM, "reload_frees", "flows freed on reload with changed memcap" },
+ { CountType::SUM, "alloc_prunes", "flows pruned on allocation of IP flows" },
+ { CountType::END, nullptr, nullptr },
+};
+
+struct PerfPegStats
+{
+ PegCount total_packets;
+ PegCount total_frees;
+ PegCount reload_frees;
+ PegCount alloc_prunes;
+};
+
+#endif
+
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// perf_reload_tuner.h author Michael Matirko <mmatirko@cisco.com>
+
+#ifndef PERF_RELOAD_TUNER_H
+#define PERF_RELOAD_TUNER_H
+
+#include "main/snort_config.h"
+
+class PerfMonReloadTuner : public snort::ReloadResourceTuner
+{
+public:
+ PerfMonReloadTuner() = default;
+
+ bool tinit() override;
+
+ bool tune_idle_context() override
+ { return tune_resources(max_work_idle); }
+
+ bool tune_packet_context() override
+ { return tune_resources(max_work); }
+
+ bool tune_resources(unsigned work_limit);
+
+ void set_memcap(size_t new_memcap)
+ { memcap = new_memcap; }
+
+ size_t get_memcap()
+ { return memcap; }
+
+private:
+ size_t memcap = 0;
+
+};
+
+#endif
+
projects / thirdparty / snort3.git / commitdiff
