hashes.h
hash_defs.h
hash_key_operations.h
+ lru_cache_local.h
lru_cache_shared.h
xhash.h
)
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// lru_cache_local.h author Masud Hasan <mashasan@cisco.com>
+
+#ifndef LRU_CACHE_LOCAL_H
+#define LRU_CACHE_LOCAL_H
+
+// LruCacheLocal - A simple thread-unsafe memcap-enforced least-recently-used cache.
+
+#include <list>
+#include <unordered_map>
+#include <vector>
+
+#include "framework/counts.h"
+
+#define LRU_CACHE_LOCAL_PEGS(module) \
+ { CountType::SUM, "cache_adds", module " cache added new entry" }, \
+ { CountType::SUM, "cache_hits", module " cache found existing entry" }, \
+ { CountType::SUM, "cache_misses", module " cache did not find entry" }, \
+ { CountType::SUM, "cache_replaces", module " cache found entry and replaced its value" }, \
+ { CountType::SUM, "cache_max", module " cache's maximum byte usage"}, \
+ { CountType::SUM, "cache_prunes", module " cache pruned entry to make space for new entry" }
+
+struct LruCacheLocalStats
+{
+ PegCount cache_adds;
+ PegCount cache_hits;
+ PegCount cache_misses;
+ PegCount cache_replaces;
+ PegCount cache_max;
+ PegCount cache_prunes;
+};
+
+template<typename Key, typename Value, typename Hash>
+class LruCacheLocal
+{
+public:
+ LruCacheLocal(const size_t sz, struct LruCacheLocalStats& st)
+ : max_size(sz), current_size(0), stats(st) { }
+
+ virtual ~LruCacheLocal() = default;
+
+ // Return 1 if an entry associated with the key exists, else return 0
+ int count(const Key&);
+
+ // Return the entry associated with the key; insert new entry if absent
+ Value& find_else_create(const Key&, bool* is_new = nullptr);
+
+ // If key does not exist, insert the key-value pair and return true;
+ // else return false replacing the existing value if asked
+ bool add(const Key&, const Value&, bool replace = false);
+
+ // Copy all key-value pairs from the cache
+ void get_all_values(std::vector<std::pair<Key, Value>>&);
+
+protected:
+ using LruList = std::list<std::pair<Key, Value>>;
+ using LruListIter = typename LruList::iterator;
+ using LruMap = std::unordered_map<Key, LruListIter, Hash>;
+ using LruMapIter = typename LruMap::iterator;
+
+ void prune();
+ void add_entry(const Key&, const Value&);
+
+ static constexpr size_t entry_size = 2 * sizeof(Key) + sizeof(Value) + sizeof(LruListIter);
+ const size_t max_size;
+ size_t current_size;
+ LruList list;
+ LruMap map;
+ struct LruCacheLocalStats& stats;
+};
+
+template<typename Key, typename Value, typename Hash>
+void LruCacheLocal<Key, Value, Hash>::prune()
+{
+ if ( !max_size )
+ return;
+
+ while ( current_size > max_size and !list.empty() )
+ {
+ auto it = --list.end();
+ map.erase(it->first);
+ list.erase(it);
+ current_size -= entry_size;
+ ++stats.cache_prunes;
+ }
+}
+
+template<typename Key, typename Value, typename Hash>
+int LruCacheLocal<Key, Value, Hash>::count(const Key& key)
+{
+ return map.count(key);
+}
+
+template<typename Key, typename Value, typename Hash>
+void LruCacheLocal<Key, Value, Hash>::add_entry(const Key& key, const Value& value)
+{
+ stats.cache_adds++;
+ list.emplace_front(std::make_pair(key, value));
+ map[key] = list.begin();
+ current_size += entry_size;
+ prune();
+ if ( stats.cache_max < current_size )
+ stats.cache_max = current_size;
+}
+
+template<typename Key, typename Value, typename Hash>
+Value& LruCacheLocal<Key, Value, Hash>::find_else_create(const Key& key, bool* is_new)
+{
+ LruMapIter it = map.find(key);
+ if (it == map.end())
+ {
+ stats.cache_misses++;
+ add_entry(key, Value());
+ if ( is_new )
+ *is_new = true;
+ return list.begin()->second;
+ }
+
+ stats.cache_hits++;
+ list.splice(list.begin(), list, it->second);
+ return list.begin()->second;
+}
+
+template<typename Key, typename Value, typename Hash>
+bool LruCacheLocal<Key, Value, Hash>::add(const Key& key, const Value& value, bool replace)
+{
+ LruMapIter it = map.find(key);
+ if (it == map.end())
+ {
+ stats.cache_misses++;
+ add_entry(key, value);
+ return true;
+ }
+
+ stats.cache_hits++;
+ list.splice(list.begin(), list, it->second);
+ if ( replace )
+ {
+ it->second->second = value;
+ stats.cache_replaces++;
+ }
+ return false;
+}
+
+template<typename Key, typename Value, typename Hash>
+void LruCacheLocal<Key, Value, Hash>::get_all_values(std::vector<std::pair<Key, Value>>& kv)
+{
+ for (auto& entry : list )
+ kv.emplace_back(entry);
+}
+
+#endif
+add_cpputest( lru_cache_local_test
+ SOURCES ../lru_cache_local.h
+)
+
add_cpputest( lru_cache_shared_test
SOURCES ../lru_cache_shared.cc
)
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// lru_cache_local.h author Masud Hasan <mashasan@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "hash/lru_cache_local.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+
+TEST_GROUP(lru_cache_local)
+{
+};
+
+TEST(lru_cache_local, basic)
+{
+ LruCacheLocalStats stats;
+ LruCacheLocal<int, int, std::hash<int>> lru_cache(40, stats);
+
+ // Check pruning in LRU order; memcap = 40 bytes would hold 2 entries
+ int key = 1;
+ int val = 100;
+ lru_cache.add(key, val);
+
+ key = 2;
+ val = 200;
+ lru_cache.add(key, val);
+
+ key = 3;
+ val = 300;
+ lru_cache.add(key, val);
+
+ // Check entries are stored in LRU order indeed
+ std::vector<std::pair<int, int>> vec;
+ lru_cache.get_all_values(vec);
+ CHECK(vec.size() == 2);
+ CHECK(vec[0].first == 3 and vec[0].second == 300);
+ CHECK(vec[1].first == 2 and vec[1].second == 200);
+
+ // Check non-existent entry; find_else_create() would return a new entry
+ auto& entry = lru_cache.find_else_create(4);
+ entry = 400;
+ // Subsequent calls would return the same one
+ CHECK(lru_cache.find_else_create(4) == 400);
+ entry = 4000;
+ CHECK(lru_cache.find_else_create(4) == 4000);
+
+ // The cache is changed after the above calls
+ vec.clear();
+ lru_cache.get_all_values(vec);
+ CHECK(vec.size() == 2);
+ CHECK(vec[0].first == 4 and vec[0].second == 4000);
+ CHECK(vec[1].first == 3 and vec[1].second == 300);
+}
+
+int main(int argc, char** argv)
+{
+ return CommandLineTestRunner::RunAllTests(argc, argv);
+}
set (HOST_TRACKER_INCLUDES
+ cache_allocator.h
+ cache_interface.h
host_cache.h
- host_cache_allocator.h
- host_cache_interface.h
host_tracker.h
)
add_library( host_tracker OBJECT
${HOST_TRACKER_INCLUDES}
+ cache_allocator.cc
host_cache.cc
- host_cache_allocator.cc
host_cache_module.cc
host_cache_module.h
host_tracker_module.cc
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_cache_allocator.cc author Silviu Minut <sminut@cisco.com>
+// cache_allocator.cc author Silviu Minut <sminut@cisco.com>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
-#ifndef HOST_CACHE_ALLOCATOR_CC
-#define HOST_CACHE_ALLOCATOR_CC
+#ifndef CACHE_ALLOCATOR_CC
+#define CACHE_ALLOCATOR_CC
#include "host_cache.h"
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_cache_allocator.h author Silviu Minut <sminut@cisco.com>
+// cache_allocator.h author Silviu Minut <sminut@cisco.com>
-#ifndef HOST_CACHE_ALLOCATOR_H
-#define HOST_CACHE_ALLOCATOR_H
+#ifndef CACHE_ALLOCATOR_H
+#define CACHE_ALLOCATOR_H
#include <cassert>
-#include "host_cache_interface.h"
+#include "cache_interface.h"
template <class T>
-class HostCacheAlloc : public std::allocator<T>
+class CacheAlloc : public std::allocator<T>
{
public:
template <class U>
struct rebind
{
- typedef HostCacheAlloc<U> other;
+ typedef CacheAlloc<U> other;
};
T* allocate(std::size_t n);
protected:
- HostCacheInterface* lru = nullptr;
+ CacheInterface* lru = nullptr;
};
template <class T>
-T* HostCacheAlloc<T>::allocate(std::size_t n)
+T* CacheAlloc<T>::allocate(std::size_t n)
{
size_t sz = n * sizeof(T);
T* out = std::allocator<T>::allocate(n);
}
template <class T>
-void HostCacheAlloc<T>::deallocate(T* p, std::size_t n) noexcept
+void CacheAlloc<T>::deallocate(T* p, std::size_t n) noexcept
{
size_t sz = n * sizeof(T);
std::allocator<T>::deallocate(p, n);
// Trivial derived allocator, pointing to their own host cache.
-// HostCacheAllocIp has a HostCacheInterface* pointing to an lru cache
+// HostCacheAllocIp has a CacheInterface* pointing to an lru cache
// instantiated using snort::SfIp as the key. See host_cache.h.
// We can create different cache types by instantiating the lru cache using
-// different keys and derive here allocators with HostCacheInterface*
+// different keys and derive here allocators with CacheInterface*
// pointing to the appropriate lru cache object.
template <class T>
-class HostCacheAllocIp : public HostCacheAlloc<T>
+class HostCacheAllocIp : public CacheAlloc<T>
{
public:
typedef HostCacheAllocIp<U> other;
};
- using HostCacheAlloc<T>::lru;
+ using CacheAlloc<T>::lru;
HostCacheAllocIp();
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_cache_interface.h author Silviu Minut <sminut@cisco.com>
+// cache_interface.h author Silviu Minut <sminut@cisco.com>
-#ifndef HOST_CACHE_INTERFACE_H
-#define HOST_CACHE_INTERFACE_H
+#ifndef CACHE_INTERFACE_H
+#define CACHE_INTERFACE_H
-class HostCacheInterface
+class CacheInterface
{
public:
Allocator into a .h and a .cc file. We include the .h file in HostTracker
and declare HostCache extern only in the .cc file. Then, we have to include
the .cc file also in the HostTracker implementation file because Allocator
-is templated. See host_cache.h, host_cache_allocator.h, host_cache_allocator.cc,
+is templated. See host_cache.h, cache_allocator.h, cache_allocator.cc,
host_tracker.h and host_tracker.cc.
-Illustrative examples are test/host_cache_allocator_test.cc (standalone
+Illustrative examples are test/cache_allocator_test.cc (standalone
host cache / allocator example) and test/host_cache_allocator_ht_test.cc
(host_cache / allocator with host tracker example).
#include <cassert>
#include "hash/lru_cache_shared.h"
-#include "host_cache_interface.h"
-#include "host_cache_allocator.h"
#include "host_tracker.h"
#include "log/messages.h"
#include "main/snort_config.h"
#include "sfip/sf_ip.h"
#include "utils/stats.h"
+#include "cache_allocator.h"
+#include "cache_interface.h"
+
// Used to create hash of key for indexing into cache.
//
// Note that both HashIp and IpEqualTo below ignore the IP family.
template<typename Key, typename Value, typename Hash, typename Eq = std::equal_to<Key>,
typename Purgatory = std::vector<std::shared_ptr<Value>>>
class LruCacheSharedMemcap : public LruCacheShared<Key, Value, Hash, Eq, Purgatory>,
- public HostCacheInterface
+ public CacheInterface
{
public:
using LruBase = LruCacheShared<Key, Value, Hash, Eq, Purgatory>;
#include "network_inspectors/rna/rna_flow.h"
#include "utils/util.h"
+#include "cache_allocator.cc"
#include "host_cache.h"
-#include "host_cache_allocator.cc"
#include "host_tracker.h"
using namespace snort;
#include <vector>
#include "framework/counts.h"
-#include "host_cache_allocator.h"
#include "main/snort_types.h"
#include "main/thread.h"
#include "network_inspectors/appid/application_ids.h"
#include "protocols/vlan.h"
#include "time/packet_time.h"
+#include "cache_allocator.h"
+
struct HostTrackerStats
{
PegCount service_adds;
#endif
#include "host_tracker_module.h"
-#include "host_cache_allocator.cc"
#include "log/messages.h"
#include "main/snort_config.h"
+#include "cache_allocator.cc"
+
using namespace snort;
const PegInfo host_tracker_pegs[] =
#include <cassert>
#include "framework/module.h"
+#include "host_tracker/cache_allocator.cc"
#include "host_tracker/host_cache.h"
-#include "host_tracker/host_cache_allocator.cc"
#define host_tracker_help \
"configure hosts"
../../sfip/sf_ip.cc
)
-add_cpputest( host_cache_allocator_test
+add_cpputest( cache_allocator_test
SOURCES
../host_tracker.cc
../../network_inspectors/rna/test/rna_flow_stubs.cc
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_cache_allocator_test.cc author Silviu Minut <sminut@cisco.com>
+// cache_allocator_test.cc author Silviu Minut <sminut@cisco.com>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "host_tracker/host_cache.h"
-#include "host_tracker/host_cache_allocator.cc"
+#include "host_tracker/cache_allocator.cc"
#include "network_inspectors/rna/rna_flow.h"
#include <string>
using namespace std;
using namespace snort;
-// Derive an allocator from HostCacheAlloc:
+// Derive an allocator from CacheAlloc:
template <class T>
-class Allocator : public HostCacheAlloc<T>
+class Allocator : public CacheAlloc<T>
{
public:
typedef Allocator<U> other;
};
- using HostCacheAlloc<T>::lru;
+ using CacheAlloc<T>::lru;
Allocator();
};
CacheType cache(100);
// Implement the allocator constructor AFTER we have a cache object
-// to point to and the implementation of our base HostCacheAlloc:
+// to point to and the implementation of our base CacheAlloc:
template <class T>
Allocator<T>::Allocator()
{
#include "config.h"
#endif
+#include "host_tracker/cache_allocator.cc"
#include "host_tracker/host_cache.h"
-#include "host_tracker/host_cache_allocator.cc"
#include "network_inspectors/rna/rna_flow.h"
#include <cstring>
#include <cstring>
+#include "host_tracker/cache_allocator.cc"
#include "host_tracker/host_cache.h"
-#include "host_tracker/host_cache_allocator.cc"
#include "network_inspectors/rna/rna_flow.h"
#include <CppUTest/CommandLineTestRunner.h>
#include <pcre.h>
#include <unordered_map>
+#include "host_tracker/cache_allocator.cc"
+#include "host_tracker/host_cache.h"
#include "log/messages.h"
#include "main/snort_debug.h"
#include "main/snort_types.h"
#include "lua_detector_util.h"
#include "service_plugins/service_discovery.h"
#include "service_plugins/service_ssl.h"
-#include "host_tracker/host_cache.h"
-#include "host_tracker/host_cache_allocator.cc"
using namespace snort;
using namespace std;
#include "hash/hash_key_operations.h"
// Non-standard, required to include template definition across compiler translation units
-#include "host_tracker/host_cache_allocator.cc"
-#include "host_tracker/host_cache_allocator.h"
+#include "host_tracker/cache_allocator.cc"
+#include "host_tracker/cache_allocator.h"
#define MAC_CACHE_INITIAL_SIZE 1024 * 1024 // Default to 1 MB
template <class T>
-class HostCacheAllocMac : public HostCacheAlloc<T>
+class HostCacheAllocMac : public CacheAlloc<T>
{
public:
template <class U>
typedef HostCacheAllocMac<U> other;
};
- using HostCacheAlloc<T>::lru;
+ using CacheAlloc<T>::lru;
HostCacheAllocMac();
};
set ( FILE_LIST
+ netflow_cache.h
netflow_headers.h
netflow_module.cc
netflow_module.h
#include "config.h"
#endif
-#include "netflow_headers.h"
-#include "netflow_module.h"
-
#include <fstream>
#include <mutex>
#include <sys/stat.h>
#include "profiler/profiler.h"
#include "protocols/packet.h"
#include "pub_sub/netflow_event.h"
-#include "sfip/sf_ip.h"
#include "src/utils/endian.h"
#include "utils/util.h"
+#include "netflow_cache.cc"
+
using namespace snort;
THREAD_LOCAL NetflowStats netflow_stats;
THREAD_LOCAL ProfileStats netflow_perf_stats;
-// compare struct to use with ip sort
-struct IpCompare
-{
- bool operator()(const snort::SfIp& a, const snort::SfIp& b)
- { return a.less_than(b); }
-};
-
// Used to ensure we fully populate the record; can't rely on the actual values being zero
struct RecordStatus
{
// static variables
// -----------------------------------------------------------------------------
-// Used to avoid creating multiple events for the same initiator IP.
-// Cache can be thread local since Netflow packets coming from a Netflow
-// device will go to the same thread.
-typedef std::unordered_map<snort::SfIp, NetflowSessionRecord, NetflowHash> NetflowCache;
-static THREAD_LOCAL NetflowCache* netflow_cache = nullptr;
+// temporary cache required to dump the output
+typedef std::pair<snort::SfIp, NetflowSessionRecord> IpRecord;
+typedef std::vector<IpRecord> DumpCache;
+static DumpCache* dump_cache = nullptr;
-// cache required to dump the output
-static NetflowCache* dump_cache = nullptr;
-
-// Netflow version 9 Template fields cache.
-typedef std::unordered_map<std::pair<uint16_t, snort::SfIp>, std::vector<Netflow9TemplateField>, TemplateIpHash> TemplateFieldCache;
-static THREAD_LOCAL TemplateFieldCache* template_cache = nullptr;
+// compare struct to use with ip sort
+struct IpCompare
+{
+ bool operator()(const IpRecord& a, const IpRecord& b)
+ { return a.first.less_than(b.first); }
+};
// -----------------------------------------------------------------------------
// static functions
// -----------------------------------------------------------------------------
+
static bool filter_record(const NetflowRules* rules, const int zone,
const SfIp* src, const SfIp* dst)
{
}
static bool version_9_record_update(const unsigned char* data, uint32_t unix_secs,
- std::vector<Netflow9TemplateField>::iterator field, NetflowSessionRecord &record,
+ uint16_t field_type, uint16_t field_length, NetflowSessionRecord &record,
RecordStatus& record_status)
{
- switch ( field->field_type )
+ switch ( field_type )
{
case NETFLOW_PROTOCOL:
// invalid protocol
- if( field->field_length != sizeof(record.proto) )
+ if( field_length != sizeof(record.proto) )
return false;
record.proto = (uint8_t)*data;
case NETFLOW_TCP_FLAGS:
// invalid tcp flags
- if( field->field_length != sizeof(record.tcp_flags ) )
+ if( field_length != sizeof(record.tcp_flags ) )
return false;
record.tcp_flags = (uint8_t)*data;
case NETFLOW_SRC_PORT:
// invalid src port
- if( field->field_length != sizeof(record.initiator_port) )
+ if( field_length != sizeof(record.initiator_port) )
return false;
record.initiator_port = ntohs(*(const uint16_t*) data);
case NETFLOW_SRC_IP:
// invalid source ip
- if( field->field_length != sizeof(uint32_t) )
+ if( field_length != sizeof(uint32_t) )
return false;
// Invalid source IP address provided
case NETFLOW_DST_PORT:
// invalid destination port
- if( field->field_length != sizeof(record.responder_port) )
+ if( field_length != sizeof(record.responder_port) )
return false;
record.responder_port = ntohs(*(const uint16_t*) data);
case NETFLOW_DST_IP:
// invalid length
- if( field->field_length != sizeof(uint32_t) )
+ if( field_length != sizeof(uint32_t) )
return false;
// Invalid destination IP address
case NETFLOW_IPV4_NEXT_HOP:
// invalid length
- if( field->field_length != sizeof(uint32_t) )
+ if( field_length != sizeof(uint32_t) )
return false;
// Invalid next-hop IP address
case NETFLOW_LAST_PKT:
- if( field->field_length != sizeof(record.last_pkt_second) )
+ if( field_length != sizeof(record.last_pkt_second) )
return false;
record.last_pkt_second = unix_secs + ntohl(*(const time_t*)data)/1000;
case NETFLOW_FIRST_PKT:
- if( field->field_length != sizeof(record.first_pkt_second) )
+ if( field_length != sizeof(record.first_pkt_second) )
return false;
record.first_pkt_second = unix_secs + ntohl(*(const time_t*)data)/1000;
case NETFLOW_IN_BYTES:
- if ( field->field_length == sizeof(uint64_t) )
+ if ( field_length == sizeof(uint64_t) )
record.initiator_bytes = ntohll(*(const uint64_t*)data);
- else if ( field->field_length == sizeof(uint32_t) )
+ else if ( field_length == sizeof(uint32_t) )
record.initiator_bytes = (uint64_t)ntohl(*(const uint32_t*)data);
- else if ( field->field_length == sizeof(uint16_t) )
+ else if ( field_length == sizeof(uint16_t) )
record.initiator_bytes = (uint64_t)ntohs(*(const uint16_t*) data);
else
return false;
case NETFLOW_IN_PKTS:
- if ( field->field_length == sizeof(uint64_t) )
+ if ( field_length == sizeof(uint64_t) )
record.initiator_pkts = ntohll(*(const uint64_t*)data);
- else if ( field->field_length == sizeof(uint32_t) )
+ else if ( field_length == sizeof(uint32_t) )
record.initiator_pkts = (uint64_t)ntohl(*(const uint32_t*)data);
- else if ( field->field_length == sizeof(uint16_t) )
+ else if ( field_length == sizeof(uint16_t) )
record.initiator_pkts = (uint64_t)ntohs(*(const uint16_t*) data);
else
return false;
case NETFLOW_SRC_TOS:
- if( field->field_length != sizeof(record.nf_src_tos) )
+ if( field_length != sizeof(record.nf_src_tos) )
return false;
record.nf_src_tos = (uint8_t)*data;
case NETFLOW_DST_TOS:
- if( field->field_length != sizeof(record.nf_dst_tos))
+ if( field_length != sizeof(record.nf_dst_tos))
return false;
record.nf_dst_tos = (uint8_t)*data;
case NETFLOW_SNMP_IN:
- if ( field->field_length == sizeof(uint32_t) )
+ if ( field_length == sizeof(uint32_t) )
record.nf_snmp_in = ntohl(*(const uint32_t*)data);
- else if ( field->field_length == sizeof(uint16_t) )
+ else if ( field_length == sizeof(uint16_t) )
record.nf_snmp_in = (uint32_t)ntohs(*(const uint16_t*) data);
else
return false;
case NETFLOW_SNMP_OUT:
- if ( field->field_length == sizeof(uint32_t) )
+ if ( field_length == sizeof(uint32_t) )
record.nf_snmp_out = ntohl(*(const uint32_t*)data);
- else if ( field->field_length == sizeof(uint16_t) )
+ else if ( field_length == sizeof(uint16_t) )
record.nf_snmp_out = (uint32_t)ntohs(*(const uint16_t*) data);
else
return false;
case NETFLOW_SRC_AS:
- if( field->field_length == sizeof(uint16_t) )
+ if( field_length == sizeof(uint16_t) )
record.nf_src_as = (uint32_t)ntohs(*(const uint16_t*) data);
- else if( field->field_length == sizeof(uint32_t) )
+ else if( field_length == sizeof(uint32_t) )
record.nf_src_as = ntohl(*(const uint32_t*)data);
else
return false;
case NETFLOW_DST_AS:
- if( field->field_length == sizeof(uint16_t) )
+ if( field_length == sizeof(uint16_t) )
record.nf_dst_as = (uint32_t)ntohs(*(const uint16_t*) data);
- else if( field->field_length == sizeof(uint32_t) )
+ else if( field_length == sizeof(uint32_t) )
record.nf_dst_as = ntohl(*(const uint32_t*)data);
else
return false;
case NETFLOW_SRC_MASK:
case NETFLOW_SRC_MASK_IPV6:
- if( field->field_length != sizeof(record.nf_src_mask) )
+ if( field_length != sizeof(record.nf_src_mask) )
return false;
record.nf_src_mask = (uint8_t)*data;
case NETFLOW_DST_MASK:
case NETFLOW_DST_MASK_IPV6:
- if( field->field_length != sizeof(record.nf_dst_mask) )
+ if( field_length != sizeof(record.nf_dst_mask) )
return false;
record.nf_dst_mask = (uint8_t)*data;
// It's a data flowset
if ( f_id > 255 && template_cache->count(ti_key) > 0 )
{
- std::vector<Netflow9TemplateField> tf;
- tf = template_cache->at(ti_key);
+ auto& tf = template_cache->find_else_create(ti_key);
while( data < flowset_end && records )
{
if ( !bad_field )
{
bool status = version_9_record_update(data, header.unix_secs,
- t_field, record, record_status);
+ t_field->field_type, t_field->field_length, record, record_status);
if ( !status )
bad_field = true;
DataBus::publish(NETFLOW_EVENT, event);
}
- // check if record exists
- auto result = netflow_cache->find(record.initiator_ip);
-
- if ( result != netflow_cache->end() )
- {
- // record exists and hence first remove the element
- netflow_cache->erase(record.initiator_ip);
- --netflow_stats.unique_flows;
- }
-
- // emplace doesn't replace element if exist, hence removing it first
- netflow_cache->emplace(record.initiator_ip, record);
- ++netflow_stats.unique_flows;
+ if ( netflow_cache->add(record.initiator_ip, record, true) )
+ ++netflow_stats.unique_flows;
records--;
}
if ( field_count > 0 )
{
- auto t_key = std::make_pair(t_id, device_ip);
-
- // remove if there any entry exists for this template
- auto is_erased = template_cache->erase(t_key);
-
- // count only unique templates
- if ( is_erased == 1 )
- --netflow_stats.v9_templates;
-
- // add template to cache
- template_cache->emplace(t_key, tf);
-
- // update the total templates count
- ++netflow_stats.v9_templates;
+ if ( template_cache->insert(std::make_pair(t_id, device_ip), tf) )
+ ++netflow_stats.v9_templates;
// don't count template as record
netflow_stats.records--;
record.nf_src_mask = precord->src_mask;
record.nf_dst_mask = precord->dst_mask;
- // insert record
- auto result = netflow_cache->emplace(record.initiator_ip, record);
-
- // new unique record
- if ( result.second )
+ if ( netflow_cache->add(record.initiator_ip, record, false) )
++netflow_stats.unique_flows;
// send create_host and create_service flags too so that rna event handler can log those
void NetflowInspector::show(const SnortConfig*) const
{
+ ConfigLogger::log_value("flow_memcap", config->flow_memcap);
+ ConfigLogger::log_value("template_memcap", config->template_memcap);
ConfigLogger::log_value("dump_file", config->dump_file);
ConfigLogger::log_value("update_timeout", config->update_timeout);
bool log_header = true;
void NetflowInspector::stringify(std::ofstream& file_stream)
{
- std::vector<snort::SfIp> keys;
- keys.reserve(dump_cache->size());
-
- for (const auto& elem : *dump_cache)
- keys.push_back(elem.first);
-
- std::sort(keys.begin(),keys.end(), IpCompare());
+ std::sort(dump_cache->begin(), dump_cache->end(), IpCompare());
std::string str;
SfIpString ip_str;
uint32_t i = 0;
- auto& cache = *dump_cache;
-
- for (auto elem : keys)
+ for (auto& elem : *dump_cache)
{
- NetflowSessionRecord& record = cache[elem];
+ NetflowSessionRecord& record = elem.second;
str = "Netflow Record #";
str += std::to_string(++i);
str += "\n";
str += " Initiator IP (Port): ";
- str += elem.ntop(ip_str);
+ str += elem.first.ntop(ip_str);
str += " (" + std::to_string(record.initiator_port) + ")";
str += " -> Responder IP (Port): ";
{
// create dump cache
if ( ! dump_cache )
- dump_cache = new NetflowCache;
+ dump_cache = new DumpCache;
}
}
void NetflowInspector::tinit()
{
if ( !netflow_cache )
- netflow_cache = new NetflowCache;
+ netflow_cache = new NetflowCache(config->flow_memcap, netflow_stats);
if ( !template_cache )
- template_cache = new TemplateFieldCache;
-
+ template_cache = new TemplateFieldCache(config->template_memcap, netflow_stats);
}
void NetflowInspector::tterm()
{
static std::mutex stats_mutex;
std::lock_guard<std::mutex> lock(stats_mutex);
- {
- // insert each cache
- dump_cache->insert(netflow_cache->begin(), netflow_cache->end());
- }
+ netflow_cache->get_all_values(*dump_cache);
}
delete netflow_cache;
delete template_cache;
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// netflow_cache.cc author Masud Hasan <mashasan@cisco.com>
+
+#ifndef NETFLOW_CACHE_CC
+#define NETFLOW_CACHE_CC
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "netflow_cache.h"
+
+THREAD_LOCAL NetflowCache* netflow_cache = nullptr;
+
+template <class T>
+LruCacheAllocNetflow<T>::LruCacheAllocNetflow()
+{
+ lru = netflow_cache;
+}
+
+THREAD_LOCAL TemplateFieldCache* template_cache = nullptr;
+
+template <class T>
+LruCacheAllocTemplate<T>::LruCacheAllocTemplate()
+{
+ lru = template_cache;
+}
+
+#endif
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// netflow_cache.h author Masud Hasan <mashasan@cisco.com>
+
+#ifndef NETFLOW_CACHE_H
+#define NETFLOW_CACHE_H
+
+#include <cassert>
+
+#include "hash/lru_cache_local.h"
+#include "host_tracker/cache_allocator.h"
+#include "sfip/sf_ip.h"
+
+#include "netflow_headers.h"
+#include "netflow_module.h"
+
+// Trivial derived allocator, pointing to their own cache. LruCacheAllocNetflow has a
+// CacheInterface* pointing to an lru cache. We can create different cache types by
+// instantiating the lru cache using different keys and derive here allocators with
+// CacheInterface* pointing to the appropriate lru cache object.
+template <class T>
+class LruCacheAllocNetflow : public CacheAlloc<T>
+{
+public:
+ // This needs to be in every derived class:
+ template <class U>
+ struct rebind
+ {
+ typedef LruCacheAllocNetflow<U> other;
+ };
+
+ using CacheAlloc<T>::lru;
+ LruCacheAllocNetflow();
+};
+
+template<typename Key, typename Value, typename Hash>
+class LruCacheLocalNetflow : public LruCacheLocal<Key, Value, Hash>, public CacheInterface
+{
+public:
+ using LruLocal = LruCacheLocal<Key, Value, Hash>;
+ using LruLocal::current_size;
+ using LruLocal::max_size;
+ using LruLocal::list;
+
+ LruCacheLocalNetflow(const size_t sz, struct LruCacheLocalStats& st) : LruLocal(sz, st) {}
+
+ template <class T>
+ friend class LruCacheAllocNetflow;
+
+private:
+ // Only the allocator calls this
+ void update(int size) override
+ {
+ if ( size < 0 )
+ assert( current_size >= (size_t) -size);
+
+ // Checking 1+ size prevents crash if max_size is too low to hold even a single entry
+ if ( (current_size += size) > max_size and list.size() > 1 )
+ LruLocal::prune();
+ }
+};
+
+template <class T>
+class LruCacheAllocTemplate : public CacheAlloc<T>
+{
+public:
+ template <class U>
+ struct rebind
+ {
+ typedef LruCacheAllocTemplate<U> other;
+ };
+
+ using CacheAlloc<T>::lru;
+ LruCacheAllocTemplate();
+};
+
+template<typename Key, typename Value, typename Hash>
+class LruCacheLocalTemplate : public LruCacheLocal<Key, Value, Hash>, public CacheInterface
+{
+public:
+ using LruLocal = LruCacheLocal<Key, Value, Hash>;
+ using LruLocal::current_size;
+ using LruLocal::max_size;
+ using LruLocal::stats;
+ using LruLocal::list;
+
+ LruCacheLocalTemplate(const size_t sz, struct LruCacheLocalStats& st) : LruLocal(sz, st)
+ {}
+
+ bool insert(const Key& key, std::vector<Netflow9TemplateField>& tf)
+ {
+ bool is_new = false;
+ Value& entry = LruLocal::find_else_create(key, &is_new);
+
+ if ( !is_new )
+ {
+ stats.cache_replaces++;
+ entry.clear();
+ }
+
+ for ( auto& elem : tf )
+ entry.emplace_back(elem.field_type, elem.field_length);
+
+ return is_new;
+ }
+
+ template <class T>
+ friend class LruCacheAllocTemplate;
+
+private:
+ void update(int size) override
+ {
+ if ( size < 0 )
+ assert( current_size >= (size_t) -size);
+
+ if ( (current_size += size) > max_size and list.size() > 1 )
+ LruLocal::prune();
+ }
+};
+
+// Used to track record for unique IP; we assume Netflow packets coming from
+// a given Netflow device will go to the same thread
+typedef LruCacheLocalNetflow<snort::SfIp, NetflowSessionRecord, NetflowHash> NetflowCache;
+
+// Used to track Netflow version 9 Template fields
+typedef std::pair<uint16_t, snort::SfIp> TemplateFieldKey;
+typedef LruCacheAllocTemplate<Netflow9TemplateField> TemplateAllocator;
+typedef std::vector<Netflow9TemplateField, TemplateAllocator> TemplateFieldValue;
+typedef LruCacheLocalTemplate<TemplateFieldKey, TemplateFieldValue, TemplateIpHash> TemplateFieldCache;
+
+#endif
{ "rules", Parameter::PT_LIST, device_rule_params, nullptr,
"list of NetFlow device rules" },
+ { "flow_memcap", Parameter::PT_INT, "0:maxSZ", "0",
+ "maximum memory for flow record cache in bytes, 0 = unlimited" },
+
+ { "template_memcap", Parameter::PT_INT, "0:maxSZ", "0",
+ "maximum memory for template cache in bytes, 0 = unlimited" },
+
{ nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
};
static const PegInfo netflow_pegs[] =
{
+ LRU_CACHE_LOCAL_PEGS("netflow"),
{ CountType::SUM, "invalid_netflow_record", "count of invalid netflow records" },
{ CountType::SUM, "packets", "total packets processed" },
{ CountType::SUM, "records", "total records found in netflow data" },
}
bool NetflowModule::set(const char*, Value& v, SnortConfig*)
{
- if ( v.is("dump_file") )
+ if ( v.is("flow_memcap") )
+ conf->flow_memcap = v.get_size();
+ else if ( v.is("template_memcap") )
+ conf->template_memcap = v.get_size();
+ else if ( v.is("dump_file") )
{
if ( conf->dump_file )
snort_free((void*)conf->dump_file);
#include <unordered_map>
#include "framework/module.h"
+#include "hash/lru_cache_local.h"
#include "sfip/sf_cidr.h"
#include "utils/util.h"
const char* dump_file = nullptr;
std::unordered_map <snort::SfIp, NetflowRules, NetflowHash> device_rule_map;
uint32_t update_timeout = 0;
+ size_t flow_memcap = 0;
+ size_t template_memcap = 0;
};
-struct NetflowStats
+struct NetflowStats : public LruCacheLocalStats
{
PegCount invalid_netflow_record;
PegCount packets;
projects / thirdparty / snort3.git / commitdiff
