From: Mike Stepanek (mstepane) Date: Thu, 11 Jul 2019 10:51:54 +0000 (-0400) Subject: Merge pull request #1665 in SNORT/snort3 from ~MASHASAN/snort3:host_cache_dump to... X-Git-Tag: 3.0.0-258~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7716471c64ddcfd841a65bddde2e336222907a3e;p=thirdparty%2Fsnort3.git Merge pull request #1665 in SNORT/snort3 from ~MASHASAN/snort3:host_cache_dump to master Squashed commit of the following: commit 363786e572c5274704c3c34355e5e01c694082ca Author: Masud Hasan Date: Wed Jul 3 09:08:41 2019 -0400 host_cache: Adding command and config option to dump hosts --- diff --git a/src/host_tracker/host_cache.h b/src/host_tracker/host_cache.h index 7a89dad1d..9b96038fa 100644 --- a/src/host_tracker/host_cache.h +++ b/src/host_tracker/host_cache.h @@ -29,25 +29,26 @@ #include "hash/lru_cache_shared.h" #include "host_tracker/host_tracker.h" +#define HOST_IP_KEY_SIZE 16 + struct HostIpKey { - static const int key_size = 16; union host_ip_addr { - uint8_t ip8[key_size]; - uint64_t ip64[key_size/8]; + uint8_t ip8[HOST_IP_KEY_SIZE]; + uint64_t ip64[HOST_IP_KEY_SIZE/8]; } ip_addr = {{0}}; // Holds either IPv4 or IPv6 addr HostIpKey() = default; - HostIpKey(const uint8_t ip[key_size]) + HostIpKey(const uint8_t ip[HOST_IP_KEY_SIZE]) { - memcpy(&ip_addr, ip, key_size); + memcpy(&ip_addr, ip, HOST_IP_KEY_SIZE); } inline bool operator==(const HostIpKey& rhs) const { - return !memcmp(&ip_addr, &rhs.ip_addr, key_size); + return !memcmp(&ip_addr, &rhs.ip_addr, HOST_IP_KEY_SIZE); } }; diff --git a/src/host_tracker/host_cache_module.cc b/src/host_tracker/host_cache_module.cc index e1c82009a..a22c4b673 100644 --- a/src/host_tracker/host_cache_module.cc +++ b/src/host_tracker/host_cache_module.cc @@ -24,12 +24,57 @@ #include "host_cache_module.h" +#include +#include +#include + +#include "log/messages.h" +#include "managers/module_manager.h" +#include "utils/util.h" + #include "host_cache.h" using namespace snort; +using namespace std; + +//------------------------------------------------------------------------- +// commands +//------------------------------------------------------------------------- + +static int host_cache_dump(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + mod->log_host_cache( luaL_optstring(L, 1, nullptr), true ); + return 0; +} + +static const Parameter host_cache_cmd_params[] = +{ + { "file_name", Parameter::PT_STRING, nullptr, nullptr, "file name to dump host cache" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; -const Parameter HostCacheModule::host_cache_params[] = +static const Command host_cache_cmds[] = { + { "dump", host_cache_dump, host_cache_cmd_params, "dump host cache"}, + { nullptr, nullptr, nullptr, nullptr } +}; + +const Command* HostCacheModule::get_commands() const +{ + return host_cache_cmds; +} + +//------------------------------------------------------------------------- +// options +//------------------------------------------------------------------------- + +static const Parameter host_cache_params[] = +{ + { "dump_file", Parameter::PT_STRING, nullptr, nullptr, + "file name to dump host cache on shutdown; won't dump by default" }, + { "size", Parameter::PT_INT, "1:max32", nullptr, "size of host cache" }, @@ -38,7 +83,9 @@ const Parameter HostCacheModule::host_cache_params[] = bool HostCacheModule::set(const char*, Value& v, SnortConfig*) { - if ( v.is("size") ) + if ( v.is("dump_file") ) + dump_file = snort_strdup(v.get_string()); + else if ( v.is("size") ) host_cache_size = v.get_uint32(); else return false; @@ -54,7 +101,7 @@ bool HostCacheModule::begin(const char*, int, SnortConfig*) bool HostCacheModule::end(const char* fqn, int, SnortConfig*) { - if ( host_cache_size && !strcmp(fqn, "host_cache") ) + if ( host_cache_size && !strcmp(fqn, HOST_CACHE_NAME) ) { host_cache.set_max_size(host_cache_size); } @@ -62,6 +109,62 @@ bool HostCacheModule::end(const char* fqn, int, SnortConfig*) return true; } +//------------------------------------------------------------------------- +// methods +//------------------------------------------------------------------------- + +HostCacheModule::HostCacheModule() : + snort::Module(HOST_CACHE_NAME, HOST_CACHE_HELP, host_cache_params) { } + +HostCacheModule::~HostCacheModule() +{ + if ( dump_file ) + { + log_host_cache(dump_file); + snort_free((void*)dump_file); + } + host_cache.clear(); +} + +void HostCacheModule::log_host_cache(const char* file_name, bool verbose) +{ + if ( !file_name ) + { + if ( verbose ) + LogMessage("File name is needed!\n"); + return; + } + + // Prevent damaging any existing file, intentionally or not + struct stat file_stat; + if ( stat(file_name, &file_stat) == 0 ) + { + if ( verbose ) + LogMessage("File %s already exists!\n", file_name); + return; + } + + ofstream out_stream(file_name); + if ( !out_stream ) + { + if ( verbose ) + LogMessage("Couldn't open %s to write!\n", file_name); + return; + } + + string str; + const auto&& lru_data = host_cache.get_all_data(); + for ( const auto& elem : lru_data ) + { + elem.second->stringify(str); + out_stream << str << endl << endl; + } + out_stream.close(); + + if ( verbose ) + LogMessage("Dumped host cache of size = %lu to %s\n", lru_data.size(), file_name); +} + const PegInfo* HostCacheModule::get_pegs() const { return host_cache.get_pegs(); } diff --git a/src/host_tracker/host_cache_module.h b/src/host_tracker/host_cache_module.h index 77b9f8cb4..e30e55b83 100644 --- a/src/host_tracker/host_cache_module.h +++ b/src/host_tracker/host_cache_module.h @@ -25,32 +25,31 @@ #include "framework/module.h" -#define host_cache_help \ - "configure hosts" +#define HOST_CACHE_NAME "host_cache" +#define HOST_CACHE_HELP "global LRU cache of host_tracker data about hosts" class HostCacheModule : public snort::Module { public: - HostCacheModule() : snort::Module("host_cache", host_cache_help, host_cache_params, true) - { - } + HostCacheModule(); + ~HostCacheModule() override; bool begin(const char*, int, snort::SnortConfig*) override; bool end(const char*, int, snort::SnortConfig*) override; bool set(const char*, snort::Value&, snort::SnortConfig*) override; + const snort::Command* get_commands() const override; const PegInfo* get_pegs() const override; PegCount* get_counts() const override; - void sum_stats(bool) override; Usage get_usage() const override { return GLOBAL; } -private: - static const snort::Parameter host_cache_params[]; - static const snort::Parameter service_params[]; + void log_host_cache(const char* file_name, bool verbose = false); +private: + const char* dump_file = nullptr; uint32_t host_cache_size; }; diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc index 4b5d5ba6d..f9eea4417 100644 --- a/src/host_tracker/host_tracker.cc +++ b/src/host_tracker/host_tracker.cc @@ -24,5 +24,170 @@ #include "host_tracker.h" +using namespace snort; +using namespace std; + THREAD_LOCAL struct HostTrackerStats host_tracker_stats; +snort::SfIp HostTracker::get_ip_addr() +{ + std::lock_guard lck(host_tracker_lock); + return ip_addr; +} + +void HostTracker::set_ip_addr(const snort::SfIp& new_ip_addr) +{ + std::lock_guard lck(host_tracker_lock); + std::memcpy(&ip_addr, &new_ip_addr, sizeof(ip_addr)); +} + +Policy HostTracker::get_stream_policy() +{ + std::lock_guard lck(host_tracker_lock); + return stream_policy; +} + +void HostTracker::set_stream_policy(const Policy& policy) +{ + std::lock_guard lck(host_tracker_lock); + stream_policy = policy; +} + +Policy HostTracker::get_frag_policy() +{ + std::lock_guard lck(host_tracker_lock); + return frag_policy; +} + +void HostTracker::set_frag_policy(const Policy& policy) +{ + std::lock_guard lck(host_tracker_lock); + frag_policy = policy; +} + +void HostTracker::add_app_mapping(Port port, Protocol proto, AppId appid) +{ + std::lock_guard lck(host_tracker_lock); + AppMapping app_map = {port, proto, appid}; + + app_mappings.push_back(app_map); +} + +AppId HostTracker::find_app_mapping(Port port, Protocol proto) +{ + std::lock_guard lck(host_tracker_lock); + for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) + { + if (it->port == port and it->proto ==proto) + { + return it->appid; + } + } + return APP_ID_NONE; +} + +bool HostTracker::find_else_add_app_mapping(Port port, Protocol proto, AppId appid) +{ + std::lock_guard lck(host_tracker_lock); + for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) + { + if (it->port == port and it->proto ==proto) + { + return false; + } + } + AppMapping app_map = {port, proto, appid}; + + app_mappings.push_back(app_map); + return true; +} + +bool HostTracker::add_service(const HostApplicationEntry& app_entry) +{ + host_tracker_stats.service_adds++; + + std::lock_guard lck(host_tracker_lock); + + auto iter = std::find(services.begin(), services.end(), app_entry); + if (iter != services.end()) + return false; // Already exists. + + services.push_front(app_entry); + return true; +} + +void HostTracker::add_or_replace_service(const HostApplicationEntry& app_entry) +{ + host_tracker_stats.service_adds++; + + std::lock_guard lck(host_tracker_lock); + + auto iter = std::find(services.begin(), services.end(), app_entry); + if (iter != services.end()) + services.erase(iter); + + services.push_front(app_entry); +} + +bool HostTracker::find_service(Protocol ipproto, Port port, HostApplicationEntry& app_entry) +{ + HostApplicationEntry tmp_entry(ipproto, port, UNKNOWN_PROTOCOL_ID); + host_tracker_stats.service_finds++; + + std::lock_guard lck(host_tracker_lock); + + auto iter = std::find(services.begin(), services.end(), tmp_entry); + if (iter != services.end()) + { + app_entry = *iter; + return true; + } + + return false; +} + +bool HostTracker::remove_service(Protocol ipproto, Port port) +{ + HostApplicationEntry tmp_entry(ipproto, port, UNKNOWN_PROTOCOL_ID); + host_tracker_stats.service_removes++; + + std::lock_guard lck(host_tracker_lock); + + auto iter = std::find(services.begin(), services.end(), tmp_entry); + if (iter != services.end()) + { + services.erase(iter); + return true; // Assumes only one matching entry. + } + + return false; +} + +void HostTracker::stringify(string& str) +{ + str = "IP: "; + SfIpString ip_str; + str += ip_addr.ntop(ip_str); + + if ( !app_mappings.empty() ) + { + str += "\napp_mappings size: " + to_string(app_mappings.size()); + for ( const auto& elem : app_mappings ) + str += "\n port: " + to_string(elem.port) + + ", proto: " + to_string(elem.proto) + + ", appid: " + to_string(elem.appid); + } + + if ( stream_policy or frag_policy ) + str += "\nstream policy: " + to_string(stream_policy) + + ", frag policy: " + to_string(frag_policy); + + if ( !services.empty() ) + { + str += "\nservices size: " + to_string(services.size()); + for ( const auto& elem : services ) + str += "\n port: " + to_string(elem.port) + + ", proto: " + to_string(elem.ipproto) + + ", snort proto: " + to_string(elem.snort_protocol_id); + } +} diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h index ca665af67..eafc9fbdc 100644 --- a/src/host_tracker/host_tracker.h +++ b/src/host_tracker/host_tracker.h @@ -82,173 +82,55 @@ struct AppMapping class HostTracker { -private: - std::mutex host_tracker_lock; // Ensure that updates to a - // shared object are safe. - - // FIXIT-M do we need to use a host_id instead of SfIp as in sfrna? - snort::SfIp ip_addr; - std::vector< AppMapping > app_mappings; - - // Policies to apply to this host. - Policy stream_policy = 0; - Policy frag_policy = 0; - - std::list services; - std::list clients; - public: HostTracker() - { - memset(&ip_addr, 0, sizeof(ip_addr)); - } + { memset(&ip_addr, 0, sizeof(ip_addr)); } HostTracker(const snort::SfIp& new_ip_addr) - { - std::memcpy(&ip_addr, &new_ip_addr, sizeof(ip_addr)); - } - - snort::SfIp get_ip_addr() - { - std::lock_guard lck(host_tracker_lock); - return ip_addr; - } - - void set_ip_addr(const snort::SfIp& new_ip_addr) - { - std::lock_guard lck(host_tracker_lock); - std::memcpy(&ip_addr, &new_ip_addr, sizeof(ip_addr)); - } - - Policy get_stream_policy() - { - std::lock_guard lck(host_tracker_lock); - return stream_policy; - } - - void set_stream_policy(const Policy& policy) - { - std::lock_guard lck(host_tracker_lock); - stream_policy = policy; - } - - Policy get_frag_policy() - { - std::lock_guard lck(host_tracker_lock); - return frag_policy; - } - - void set_frag_policy(const Policy& policy) - { - std::lock_guard lck(host_tracker_lock); - frag_policy = policy; - } - - void add_app_mapping(Port port, Protocol proto, AppId appid) - { - std::lock_guard lck(host_tracker_lock); - AppMapping app_map = {port, proto, appid}; - - app_mappings.push_back(app_map); - } - - AppId find_app_mapping(Port port, Protocol proto) - { - std::lock_guard lck(host_tracker_lock); - for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) - { - if (it->port == port and it->proto ==proto) - { - return it->appid; - } - } - return APP_ID_NONE; - } - - bool find_else_add_app_mapping(Port port, Protocol proto, AppId appid) - { - std::lock_guard lck(host_tracker_lock); - for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) - { - if (it->port == port and it->proto ==proto) - { - return false; - } - } - AppMapping app_map = {port, proto, appid}; - - app_mappings.push_back(app_map); - return true; - } + { std::memcpy(&ip_addr, &new_ip_addr, sizeof(ip_addr)); } + + snort::SfIp get_ip_addr(); + void set_ip_addr(const snort::SfIp& new_ip_addr); + Policy get_stream_policy(); + void set_stream_policy(const Policy& policy); + Policy get_frag_policy(); + void set_frag_policy(const Policy& policy); + void add_app_mapping(Port port, Protocol proto, AppId appid); + AppId find_app_mapping(Port port, Protocol proto); + bool find_else_add_app_mapping(Port port, Protocol proto, AppId appid); // Add host service data only if it doesn't already exist. Returns // false if entry exists already, and true if entry was added. - bool add_service(const HostApplicationEntry& app_entry) - { - host_tracker_stats.service_adds++; - - std::lock_guard lck(host_tracker_lock); - - auto iter = std::find(services.begin(), services.end(), app_entry); - if (iter != services.end()) - return false; // Already exists. - - services.push_front(app_entry); - return true; - } + bool add_service(const HostApplicationEntry& app_entry); // Add host service data if it doesn't already exist. If it does exist // replace the previous entry with the new entry. - void add_or_replace_service(const HostApplicationEntry& app_entry) - { - host_tracker_stats.service_adds++; - - std::lock_guard lck(host_tracker_lock); - - auto iter = std::find(services.begin(), services.end(), app_entry); - if (iter != services.end()) - services.erase(iter); - - services.push_front(app_entry); - } + void add_or_replace_service(const HostApplicationEntry& app_entry); // Returns true and fills in copy of HostApplicationEntry when found. // Returns false when not found. - bool find_service(Protocol ipproto, Port port, HostApplicationEntry& app_entry) - { - HostApplicationEntry tmp_entry(ipproto, port, UNKNOWN_PROTOCOL_ID); - host_tracker_stats.service_finds++; - - std::lock_guard lck(host_tracker_lock); - - auto iter = std::find(services.begin(), services.end(), tmp_entry); - if (iter != services.end()) - { - app_entry = *iter; - return true; - } - - return false; - } + bool find_service(Protocol ipproto, Port port, HostApplicationEntry& app_entry); // Removes HostApplicationEntry object associated with ipproto and port. // Returns true if entry existed. False otherwise. - bool remove_service(Protocol ipproto, Port port) - { - HostApplicationEntry tmp_entry(ipproto, port, UNKNOWN_PROTOCOL_ID); - host_tracker_stats.service_removes++; + bool remove_service(Protocol ipproto, Port port); - std::lock_guard lck(host_tracker_lock); + // This should be updated whenever HostTracker data members are changed + void stringify(std::string& str); - auto iter = std::find(services.begin(), services.end(), tmp_entry); - if (iter != services.end()) - { - services.erase(iter); - return true; // Assumes only one matching entry. - } +private: + // Ensure that updates to a shared object are safe + std::mutex host_tracker_lock; - return false; - } + // FIXIT-M do we need to use a host_id instead of SfIp as in sfrna? + snort::SfIp ip_addr; + std::vector< AppMapping > app_mappings; + + // Policies to apply to this host. + Policy stream_policy = 0; + Policy frag_policy = 0; + + std::list services; }; #endif diff --git a/src/host_tracker/test/host_cache_module_test.cc b/src/host_tracker/test/host_cache_module_test.cc index 5459e003e..519fb02fa 100644 --- a/src/host_tracker/test/host_cache_module_test.cc +++ b/src/host_tracker/test/host_cache_module_test.cc @@ -23,9 +23,12 @@ #include "config.h" #endif +#include + #include "host_tracker/host_cache_module.h" #include "host_tracker/host_cache.h" #include "main/snort_config.h" +#include "managers/module_manager.h" #include #include @@ -33,6 +36,13 @@ #include "sfip/sf_ip.h" using namespace snort; +using namespace std; + +// All tests here use the same module since host_cache is global. Creating a local module for each +// test will cause host_cache PegCount testing to be dependent on the order of running these tests. +static HostCacheModule module; +#define LOG_MAX 128 +static char logged_message[LOG_MAX+1]; namespace snort { @@ -40,8 +50,20 @@ namespace snort SnortProtocolId ProtocolReference::add(char const*) { return 0; } SnortProtocolId ProtocolReference::find(char const*) { return 0; } SnortConfig* SnortConfig::get_conf() { return nullptr; } -char* snort_strdup(const char* s) -{ return strdup(s); } +char* snort_strdup(const char* s) { return strdup(s); } +Module* ModuleManager::get_module(const char*) { return nullptr; } +void LogMessage(const char* format,...) +{ + va_list args; + va_start(args, format); + vsnprintf(logged_message, LOG_MAX, format, args); + logged_message[LOG_MAX] = '\0'; +} +} + +extern "C" +{ +const char* luaL_optlstring(lua_State*, int, const char*, size_t*) { return nullptr; } } void show_stats(PegCount*, const PegInfo*, unsigned, const char*) @@ -53,17 +75,24 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) #define FRAG_POLICY 33 #define STREAM_POLICY 100 -SfIp expected_addr; - TEST_GROUP(host_cache_module) -{ }; +{ + void setup() override + { + MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); + } + + void teardown() override + { + MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); + } +}; // Test that HostCacheModule sets up host_cache size based on config. TEST(host_cache_module, host_cache_module_test_values) { Value size_val((double)2112); Parameter size_param = { "size", Parameter::PT_INT, nullptr, nullptr, "cache size" }; - HostCacheModule module; const PegInfo* ht_pegs = module.get_pegs(); const PegCount* ht_stats = module.get_counts(); @@ -97,6 +126,22 @@ TEST(host_cache_module, host_cache_module_test_values) CHECK(2112 == host_cache.get_max_size()); } +TEST(host_cache_module, log_host_cache_messages) +{ + module.log_host_cache(nullptr, true); + STRCMP_EQUAL(logged_message, "File name is needed!\n"); + + module.log_host_cache("nowhere/host_cache.dump", true); + STRCMP_EQUAL(logged_message, "Couldn't open nowhere/host_cache.dump to write!\n"); + + module.log_host_cache("host_cache.dump", true); + STRCMP_EQUAL(logged_message, "Dumped host cache of size = 0 to host_cache.dump\n"); + + module.log_host_cache("host_cache.dump", true); + STRCMP_EQUAL(logged_message, "File host_cache.dump already exists!\n"); + remove("host_cache.dump"); +} + int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/host_tracker/test/host_tracker_test.cc b/src/host_tracker/test/host_tracker_test.cc index f7c7af435..5acd3b454 100644 --- a/src/host_tracker/test/host_tracker_test.cc +++ b/src/host_tracker/test/host_tracker_test.cc @@ -29,6 +29,7 @@ #include using namespace snort; +using namespace std; namespace snort { @@ -173,6 +174,29 @@ TEST(host_tracker, add_find_service_test) CHECK(true == ret); } +TEST(host_tracker, stringify) +{ + SfIp ip; + ip.pton(AF_INET6, "feed:dead:beef::"); + HostTracker ht(ip); + ht.add_app_mapping(80, 6, 676); + ht.add_app_mapping(443, 6, 1122); + ht.set_frag_policy(3); + HostApplicationEntry app_entry(6, 80, 10); + ht.add_service(app_entry); + string host_tracker_string; + + ht.stringify(host_tracker_string); + STRCMP_EQUAL(host_tracker_string.c_str(), + "IP: feed:dead:beef:0000:0000:0000:0000:0000\n" + "app_mappings size: 2\n" + " port: 80, proto: 6, appid: 676\n" + " port: 443, proto: 6, appid: 1122\n" + "stream policy: 0, frag policy: 3\n" + "services size: 1\n" + " port: 80, proto: 6, snort proto: 10"); +} + int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/main/snort.cc b/src/main/snort.cc index e04bd590d..a3f333e9b 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -38,7 +38,6 @@ #include "flow/ha.h" #include "framework/mpse.h" #include "helpers/process.h" -#include "host_tracker/host_cache.h" #include "ips_options/ips_options.h" #include "log/log.h" #include "log/messages.h" @@ -302,7 +301,6 @@ void Snort::term() term_signals(); IpsManager::global_term(SnortConfig::get_conf()); SFAT_Cleanup(); - host_cache.clear(); #ifdef PIGLET if ( !Piglet::piglet_mode() )