${HASH_SOURCES}
hashes.cc
lru_cache_shared.h
+ lru_cache_shared.cc
sfghash.cc
sfhashfcn.cc
sfprimetable.cc
libhash_a_SOURCES = \
hashes.cc \
+lru_cache_shared.cc \
sfghash.cc \
sfhashfcn.cc \
sfprimetable.cc sfprimetable.h \
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// lru_cache_shared.cc author Steve Chew <stechew@cisco.com>
+
+#include "hash/lru_cache_shared.h"
+
+const PegInfo lru_cache_shared_peg_names[] =
+{
+ { "lru cache adds", "lru cache added new entry" },
+ { "lru cache replaces", "lru cache replaced existing entry" },
+ { "lru cache prunes", "lru cache pruned entry to make space for new entry" },
+ { "lru cache find hits", "lru cache found entry in cache" },
+ { "lru cache find misses", "lru cache did not find entry in cache" },
+ { "lru cache removes", "lru cache found entry and removed it" },
+ { "lru cache clears", "lru cache clear API calls" },
+ { nullptr, nullptr },
+};
+
#include <unordered_map>
#include <mutex>
+#include "framework/counts.h"
+
+extern const PegInfo lru_cache_shared_peg_names[];
+
+struct LruCacheSharedStats
+{
+ PegCount adds = 0; // An insert that added new entry.
+ PegCount replaces = 0; // An insert that replaced existing entry
+ PegCount prunes = 0; // When an old entry is removed to make
+ // room for a new entry.
+ PegCount find_hits = 0; // Found entry in cache.
+ PegCount find_misses = 0; // Did not find entry in cache.
+ PegCount removes = 0; // Found entry and removed it.
+ PegCount clears = 0; // Calls to clear API.
+};
+
template<typename Key, typename Data, typename Hash>
class LruCacheShared
{
return max_size;
}
+ // Modify the maximum number of entries allowed in the cache.
+ // If the size is reduced, the oldest entries are removed.
+ bool set_max_size(size_t newsize);
+
// Add data to cache or replace data if it already exists.
void insert(const Key& key, const Data& data);
// least).
std::vector<std::pair<Key, Data> > get_all_data(void);
+ const PegInfo* get_pegs() const
+ {
+ return lru_cache_shared_peg_names;
+ }
+
+ PegCount* get_counts() const
+ {
+ return (PegCount*)&stats;
+ }
+
private:
using LruList = std::list<std::pair<Key, Data> >;
using LruListIter = typename LruList::iterator;
// least recently used at the end.
LruMap map; // Maps key to list iterator for fast lookup.
+ struct LruCacheSharedStats stats;
};
+template<typename Key, typename Data, typename Hash>
+bool LruCacheShared<Key, Data, Hash>::set_max_size(size_t newsize)
+{
+ LruListIter list_iter;
+
+ if (newsize <= 0)
+ return false; // Not allowed to set size to zero.
+
+ std::lock_guard<std::mutex> cache_lock(cache_mutex);
+
+ // Remove the oldest entries if we have to reduce cache size.
+ list_iter=list.end();
+ while (current_size > newsize)
+ {
+ list_iter--;
+ current_size--;
+ map.erase(list_iter->first);
+ list.erase(list_iter);
+ }
+
+ max_size = newsize;
+ return true;
+}
+
template<typename Key, typename Data, typename Hash>
void LruCacheShared<Key, Data, Hash>::insert(const Key& key, const Data& data)
{
current_size--;
list.erase(map_iter->second);
map.erase(map_iter);
+ stats.replaces++;
+ }
+ else
+ {
+ stats.adds++;
}
// Add key/data pair to front of list.
list_iter--;
map.erase(list_iter->first);
list.erase(list_iter);
+ stats.prunes++;
}
else
{
map_iter = map.find(key);
if (map_iter == map.end())
+ {
+ stats.find_misses++;
return false; // Key is not in LruCache.
+ }
data = map_iter->second->second;
if (update)
list.splice(list.begin(), list, map_iter->second);
+ stats.find_hits++;
return true;
}
current_size--;
list.erase(map_iter->second);
map.erase(map_iter);
+ stats.removes++;
return(true);
}
current_size--;
list.erase(map_iter->second);
map.erase(map_iter);
+ stats.removes++;
return(true);
}
}
current_size = 0;
+ stats.clears++;
}
template<typename Key, typename Data, typename Hash>
--- /dev/null
+
+add_cpputest(lru_cache_shared_test hash)
+
TESTS = $(check_PROGRAMS)
lru_cache_shared_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
-lru_cache_shared_test_LDADD = @CPPUTEST_LDFLAGS@
+lru_cache_shared_test_LDADD = ../lru_cache_shared.o @CPPUTEST_LDFLAGS@
#include <iostream>
#include <functional>
#include <unordered_map>
+#include <string.h>
#include "time/stopwatch.h"
CHECK(0 == vec.size());
}
+// Test statistics counters.
+TEST(lru_cache_shared, stats_test)
+{
+ std::string data;
+ LruCacheShared<int, std::string, std::hash<int> > lru_cache(5);
+
+ for (int i = 0; i < 10; i++)
+ {
+ lru_cache.insert(i, std::to_string(i));
+ }
+
+ lru_cache.insert(8, "new-eight"); // Replace entries.
+ lru_cache.insert(9, "new-nine");
+
+ CHECK(5 == lru_cache.size());
+
+ lru_cache.find(7, data); // Hits
+ lru_cache.find(8, data);
+ lru_cache.find(9, data);
+
+ lru_cache.remove(7);
+ lru_cache.remove(8);
+ lru_cache.remove(9, data);
+ CHECK("new-nine" == data);
+
+ lru_cache.find(8, data); // Misses now that they're removed.
+ lru_cache.find(9, data);
+
+ lru_cache.remove(100); // Removing a non-existant entry does not
+ // increase remove count.
+
+ lru_cache.clear();
+
+ PegCount* stats = lru_cache.get_counts();
+
+ CHECK(stats[0] == 10); // adds
+ CHECK(stats[1] == 2); // replaces
+ CHECK(stats[2] == 5); // prunes
+ CHECK(stats[3] == 3); // find hits
+ CHECK(stats[4] == 2); // find misses
+ CHECK(stats[5] == 3); // removes
+ CHECK(stats[6] == 1); // clears
+
+ // Check statistics names.
+ const PegInfo* pegs = lru_cache.get_pegs();
+ CHECK(!strcmp(pegs[0].name, "lru cache adds"));
+ CHECK(!strcmp(pegs[1].name, "lru cache replaces"));
+ CHECK(!strcmp(pegs[2].name, "lru cache prunes"));
+ CHECK(!strcmp(pegs[3].name, "lru cache find hits"));
+ CHECK(!strcmp(pegs[4].name, "lru cache find misses"));
+ CHECK(!strcmp(pegs[5].name, "lru cache removes"));
+ CHECK(!strcmp(pegs[6].name, "lru cache clears"));
+}
+
int main(int argc, char** argv)
{
return CommandLineTestRunner::RunAllTests(argc, argv);
add_library( host_tracker STATIC
host_cache.cc
host_cache.h
- host_module.cc
- host_module.h
+ host_cache_module.cc
+ host_cache_module.h
+ host_tracker_module.cc
+ host_tracker_module.h
host_tracker.cc
host_tracker.h
)
libhost_tracker_a_SOURCES = \
host_cache.cc \
host_cache.h \
-host_module.cc \
-host_module.h \
+host_cache_module.cc \
+host_cache_module.h \
+host_tracker_module.cc \
+host_tracker_module.h \
host_tracker.cc \
host_tracker.h
* The HostTrackerModule is used to read in initial known information about
hosts, populate HostTracker objects, and place them in the host_cache.
+* The HostCache object is a thread-safe global LRU cache. The cache is
+shared between all packet threads. It contains HostTracker objects and
+provides a way for packet threads to store and retrieve data about
+hosts as it is discovered. In the long run this cache will replace the
+current Hosts table and will be the central, shared repository for data
+about hosts.
+
+* The HostCacheModule is used to configure the HostCache's size.
+
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// host_cache_module.cc author Steve Chew <stechew@cisco.com>
+
+#include "host_cache_module.h"
+
+#include "host_cache.h"
+
+const Parameter HostCacheModule::host_cache_params[] =
+{
+ { "size", Parameter::PT_INT, nullptr, nullptr,
+ "size of host cache" },
+
+ { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
+};
+
+bool HostCacheModule::set(const char*, Value& v, SnortConfig*)
+{
+ if ( v.is("size") )
+ host_cache_size = v.get_long();
+ else
+ return false;
+
+ return true;
+}
+
+bool HostCacheModule::begin(const char*, int, SnortConfig*)
+{
+ host_cache_size = 0;
+ return true;
+}
+
+bool HostCacheModule::end(const char* fqn, int, SnortConfig*)
+{
+ if ( host_cache_size && !strcmp(fqn, "host_cache") )
+ {
+ host_cache.set_max_size(host_cache_size);
+ }
+
+ return true;
+}
+
+const PegInfo* HostCacheModule::get_pegs() const
+{ return host_cache.get_pegs(); }
+
+PegCount* HostCacheModule::get_counts() const
+{ return (PegCount*)host_cache.get_counts(); }
+
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2015-2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// host_cache_module.h author Steve Chew <stechew@cisco.com>
+
+#ifndef HOST_CACHE_MODULE_H
+#define HOST_CACHE_MODULE_H
+
+// Loads host cache configuration data.
+
+#include "framework/module.h"
+#include <assert.h>
+
+#define host_cache_help \
+ "configure hosts"
+
+class HostCacheModule : public Module
+{
+public:
+ HostCacheModule() : Module("host_cache", host_cache_help, host_cache_params, true)
+ {
+ }
+
+ const PegInfo* get_pegs() const override;
+ PegCount* get_counts() const override;
+
+ bool set(const char*, Value&, SnortConfig*) override;
+ bool begin(const char*, int, SnortConfig*) override;
+ bool end(const char*, int, SnortConfig*) override;
+
+private:
+ static const Parameter host_cache_params[];
+ static const Parameter service_params[];
+
+ uint32_t host_cache_size;
+};
+
+#endif
+
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_module.cc author Steve Chew <stechew@cisco.com>
+// host_tracker_module.cc author Steve Chew <stechew@cisco.com>
-#include "host_tracker/host_module.h"
+#include "host_tracker/host_tracker_module.h"
#include "host_tracker/host_cache.h"
#include "stream/stream_api.h"
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_module.h author Steve Chew <stechew@cisco.com>
+// host_tracker_module.h author Steve Chew <stechew@cisco.com>
-#ifndef HOST_MODULE_H
-#define HOST_MODULE_H
+#ifndef HOST_TRACKER_MODULE_H
+#define HOST_TRACKER_MODULE_H
// Loads host configuration data.
add_cpputest(host_cache_test host_tracker)
-add_cpputest(host_module_test host_tracker)
+add_cpputest(host_cache_module_test host_tracker)
+add_cpputest(host_tracker_module_test host_tracker)
add_cpputest(host_tracker_test host_tracker)
-target_link_libraries(host_module_test
+target_link_libraries(host_cache_module_test
+ ${CMAKE_BINARY_DIR}/src/framework/libframework.a
+ ${CMAKE_BINARY_DIR}/src/catch/libcatch_tests.a
+ ${CMAKE_BINARY_DIR}/src/sfip/libsfip.a
+ ${CMAKE_BINARY_DIR}/src/hash/libhash.a
+ ${DNET_LIBRARIES})
+
+target_link_libraries(host_tracker_module_test
${CMAKE_BINARY_DIR}/src/framework/libframework.a
${CMAKE_BINARY_DIR}/src/catch/libcatch_tests.a
${CMAKE_BINARY_DIR}/src/sfip/libsfip.a
AM_DEFAULT_SOURCE_EXT = .cc
check_PROGRAMS = \
-host_module_test \
+host_tracker_module_test \
+host_cache_module_test \
host_cache_test \
host_tracker_test
TESTS = $(check_PROGRAMS)
-host_module_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
+host_cache_module_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
+host_tracker_module_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
host_cache_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
host_tracker_test_CPPFLAGS = @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
-host_module_test_LDADD = ../host_module.o ../host_cache.o ../host_tracker.o ../../framework/libframework.a ../../catch/libcatch_tests.a ../../sfip/libsfip.a @CPPUTEST_LDFLAGS@
+host_cache_module_test_LDADD = ../host_cache_module.o ../host_cache.o ../host_tracker.o ../../framework/libframework.a ../../catch/libcatch_tests.a ../../sfip/libsfip.a ../../hash/libhash.a @CPPUTEST_LDFLAGS@
+host_tracker_module_test_LDADD = ../host_tracker_module.o ../host_cache.o ../host_tracker.o ../../framework/libframework.a ../../catch/libcatch_tests.a ../../sfip/libsfip.a @CPPUTEST_LDFLAGS@
host_cache_test_LDADD = ../host_cache.o ../host_tracker.o @CPPUTEST_LDFLAGS@
host_tracker_test_LDADD = ../host_tracker.o @CPPUTEST_LDFLAGS@
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// host_cache_module_test.cc author Steve Chew <stechew@cisco.com>
+// unit tests for the host module APIs
+
+#include "host_tracker/host_cache_module.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+
+#include "host_tracker/host_cache.h"
+#include "sfip/sf_ip.h"
+
+// Fake AddProtocolReference to avoid bringing in a ton of dependencies.
+int16_t AddProtocolReference(const char* protocol)
+{
+ if (!strcmp("servicename", protocol))
+ return 3;
+ if (!strcmp("tcp", protocol))
+ return 2;
+ return 1;
+}
+
+// Fake show_stats to avoid bringing in a ton of dependencies.
+void show_stats(
+ PegCount*, const PegInfo*, unsigned, const char*)
+{
+}
+
+void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*)
+{
+}
+
+#define FRAG_POLICY 33
+#define STREAM_POLICY 100
+
+sfip_t expected_addr;
+
+TEST_GROUP(host_cache_module)
+{
+ void setup()
+ {
+ }
+
+ void teardown()
+ {
+ }
+};
+
+// Test that HostCacheModule sets up host_cache size based on config.
+TEST(host_cache_module, host_cache_module_test_values)
+{
+ Value size_val((double)2112);
+ Parameter size_param = { "size", Parameter::PT_INT, nullptr, nullptr, "cache size" };
+ HostCacheModule module;
+ const PegInfo* ht_pegs = module.get_pegs();
+ const PegCount* ht_stats = module.get_counts();
+
+ CHECK(!strcmp(ht_pegs[0].name, "lru cache adds"));
+ CHECK(!strcmp(ht_pegs[1].name, "lru cache replaces"));
+ CHECK(!strcmp(ht_pegs[2].name, "lru cache prunes"));
+ CHECK(!strcmp(ht_pegs[3].name, "lru cache find hits"));
+ CHECK(!strcmp(ht_pegs[4].name, "lru cache find misses"));
+ CHECK(!strcmp(ht_pegs[5].name, "lru cache removes"));
+ CHECK(!strcmp(ht_pegs[6].name, "lru cache clears"));
+ CHECK(!ht_pegs[7].name);
+
+ CHECK(ht_stats[0] == 0);
+ CHECK(ht_stats[1] == 0);
+ CHECK(ht_stats[2] == 0);
+ CHECK(ht_stats[3] == 0);
+ CHECK(ht_stats[4] == 0);
+ CHECK(ht_stats[5] == 0);
+ CHECK(ht_stats[6] == 0);
+
+ size_val.set(&size_param);
+
+ // Set up the host_cache max size.
+ module.begin("host_cache", 0, nullptr);
+ module.set(nullptr, size_val, nullptr);
+ module.end("host_cache", 0, nullptr);
+
+ ht_stats = module.get_counts();
+ CHECK(ht_stats[0] == 0);
+
+ CHECK(2112 == host_cache.get_max_size());
+}
+
+int main(int argc, char** argv)
+{
+ return CommandLineTestRunner::RunAllTests(argc, argv);
+}
+
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//--------------------------------------------------------------------------
-// host_module_test.cc author Steve Chew <stechew@cisco.com>
+// host_tracker_module_test.cc author Steve Chew <stechew@cisco.com>
// unit tests for the host module APIs
-#include "host_tracker/host_module.h"
+#include "host_tracker/host_tracker_module.h"
#include <CppUTest/CommandLineTestRunner.h>
#include <CppUTest/TestHarness.h>
sfip_t expected_addr;
-TEST_GROUP(host_module)
+TEST_GROUP(host_tracker_module)
{
void setup()
{
}
};
-// Test that HostModules variables are set correctly.
-TEST(host_module, host_module_test_values)
+// Test that HostTrackerModule variables are set correctly.
+TEST(host_tracker_module, host_tracker_module_test_values)
{
sfip_t cached_addr;
}
-// Test that HostModules statistics are correct.
-TEST(host_module, host_module_test_stats)
+// Test that HostTrackerModule statistics are correct.
+TEST(host_tracker_module, host_tracker_module_test_stats)
{
HostIpKey host_ip_key(expected_addr.ip8);
std::shared_ptr<HostTracker> ht;
#include "filters/sfthd.h"
#include "filters/sfthreshold.h"
#include "framework/module.h"
-#include "host_tracker/host_module.h"
+#include "host_tracker/host_tracker_module.h"
+#include "host_tracker/host_cache_module.h"
#include "latency/latency_module.h"
#include "managers/module_manager.h"
#include "managers/plugin_manager.h"
ModuleManager::add_module(new AttributeTableModule);
ModuleManager::add_module(new HostsModule);
ModuleManager::add_module(new HostTrackerModule);
+ ModuleManager::add_module(new HostCacheModule);
}
projects / thirdparty / snort3.git / commitdiff
