From: Masud Hasan (mashasan) Date: Tue, 20 Oct 2020 19:22:05 +0000 (+0000) Subject: Merge pull request #2535 in SNORT/snort3 from ~SMINUT/snort3:host_cache_delete to... X-Git-Tag: 3.0.3-3~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=289ae7bdbd4c8d7fd380660e207c9c128b37e9ff;p=thirdparty%2Fsnort3.git Merge pull request #2535 in SNORT/snort3 from ~SMINUT/snort3:host_cache_delete to master Squashed commit of the following: commit 32ab85e5f1d63379315b7af44570c31b397b5f08 Author: Silviu Minut Date: Thu Oct 8 16:24:16 2020 -0400 host_cache: delete host, network protocol, transport protocol, client, service, tcp fingerprint and user agent fingerprint commands host_tracker: implement client and server delete commands --- diff --git a/src/host_tracker/host_cache_module.cc b/src/host_tracker/host_cache_module.cc index e9279b786..4c63598d5 100644 --- a/src/host_tracker/host_cache_module.cc +++ b/src/host_tracker/host_cache_module.cc @@ -47,15 +47,268 @@ static int host_cache_dump(lua_State* L) return 0; } +static int host_cache_delete_host(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + { + const char* ips = luaL_optstring(L, 1, nullptr); + if (ips == nullptr) + { + LogMessage("Usage: host_cache.delete_host(ip)\n"); + return 0; + } + + SfIp ip; + if (ip.set(ips) != SFIP_SUCCESS) + { + LogMessage("Bad ip %s\n", ips); + return 0; + } + + auto ht = host_cache.find(ip); + if (ht) + ht->set_visibility(false); + else + { + LogMessage("%s not found in host cache\n", ips); + return 0; + } + + LogMessage("host_cache_delete_host done\n"); + } + return 0; +} + +static int host_cache_delete_network_proto(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + { + const char* ips = luaL_optstring(L, 1, nullptr); + int proto = luaL_optint(L, 2, -1); + + if (ips == nullptr || proto == -1) + { + LogMessage("Usage: host_cache.delete_network_proto(ip, proto)\n"); + return 0; + } + + SfIp ip; + if (ip.set(ips) != SFIP_SUCCESS) + { + LogMessage("Bad ip %s\n", ips); + return 0; + } + + auto ht = host_cache.find(ip); + if (ht) + { + if ( !ht->set_network_proto_visibility(proto, false) ) + { + LogMessage("%d not found for host %s\n", proto, ips); + return 0; + } + } + else + { + LogMessage("%s not found in host cache\n", ips); + return 0; + } + + LogMessage("host_cache_delete_network_proto done\n"); + } + return 0; +} + +static int host_cache_delete_transport_proto(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + { + const char* ips = luaL_optstring(L, 1, nullptr); + int proto = luaL_optint(L, 2, -1); + + if (ips == nullptr || proto == -1) + { + LogMessage("Usage: host_cache.delete_transport_proto(ip, proto)\n"); + return 0; + } + + SfIp ip; + if (ip.set(ips) != SFIP_SUCCESS) + { + LogMessage("Bad ip %s\n", ips); + return 0; + } + + auto ht = host_cache.find(ip); + if (ht) + { + if ( !ht->set_xproto_visibility(proto, false) ) + { + LogMessage("%d not found for host %s\n", proto, ips); + return 0; + } + } + else + { + LogMessage("%s not found in host cache\n", ips); + return 0; + } + + LogMessage("host_cache_delete_transport_proto done\n"); + } + return 0; +} + +static int host_cache_delete_service(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + { + const char* ips = luaL_optstring(L, 1, nullptr); + int port = luaL_optint(L, 2, -1); + int proto = luaL_optint(L, 3, -1); + + if (ips == nullptr || port == -1 || proto == -1) + { + LogMessage("Usage: host_cache.delete_service(ip, port, proto).\n"); + return 0; + } + + if ( !(0 <= proto and proto < 256) ) + { + LogMessage("Protocol must be between 0 and 255.\n"); + return 0; + } + + SfIp ip; + if (ip.set(ips) != SFIP_SUCCESS) + { + LogMessage("Bad ip %s\n", ips); + return 0; + } + + auto ht = host_cache.find(ip); + if (ht) + { + if ( !ht->set_service_visibility(port, (IpProtocol)proto, false) ) + { + LogMessage("%d or %d not found for host %s\n", port, proto, ips); + return 0; + } + } + else + { + LogMessage("%s not found in host cache\n", ips); + return 0; + } + + LogMessage("host_cache_delete_service done\n"); + } + return 0; +} + +static int host_cache_delete_client(lua_State* L) +{ + HostCacheModule* mod = (HostCacheModule*) ModuleManager::get_module(HOST_CACHE_NAME); + if ( mod ) + { + const char* ips = luaL_optstring(L, 1, nullptr); + int id = luaL_optint(L, 2, -1); + int service = luaL_optint(L, 3, -1); + const char* version = luaL_optstring(L, 4, nullptr); + + if (ips == nullptr || id == -1 || service == -1) + { + LogMessage("Usage: host_cache.delete_client(ip, id, service, ).\n"); + return 0; + } + + SfIp ip; + if (ip.set(ips) != SFIP_SUCCESS) + { + LogMessage("Bad ip %s\n", ips); + return 0; + } + + auto ht = host_cache.find(ip); + if (ht) + { + HostClient hc(id, version, service); + if ( !ht->set_client_visibility(hc, false) ) + { + LogMessage("Client not found for host %s\n", ips); + return 0; + } + } + else + { + LogMessage("%s not found in host cache\n", ips); + return 0; + } + + LogMessage("host_cache_delete_client done\n"); + } + return 0; +} + static const Parameter host_cache_cmd_params[] = { { "file_name", Parameter::PT_STRING, nullptr, nullptr, "file name to dump host cache" }, { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; +static const Parameter host_cache_delete_host_params[] = +{ + { "host_ip", Parameter::PT_STRING, nullptr, nullptr, "ip address to delete" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const Parameter host_cache_delete_network_proto_params[] = +{ + { "host_ip", Parameter::PT_STRING, nullptr, nullptr, "ip of host" }, + { "proto", Parameter::PT_INT, nullptr, nullptr, "network protocol to delete" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const Parameter host_cache_delete_transport_proto_params[] = +{ + { "host_ip", Parameter::PT_STRING, nullptr, nullptr, "ip of host" }, + { "proto", Parameter::PT_INT, nullptr, nullptr, "transport protocol to delete" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const Parameter host_cache_delete_service_params[] = +{ + { "host_ip", Parameter::PT_STRING, nullptr, nullptr, "ip of host" }, + { "port", Parameter::PT_INT, nullptr, nullptr, "service port" }, + { "proto", Parameter::PT_INT, nullptr, nullptr, "service protocol" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const Parameter host_cache_delete_client_params[] = +{ + { "host_ip", Parameter::PT_STRING, nullptr, nullptr, "ip of host" }, + { "id", Parameter::PT_INT, nullptr, nullptr, "application id" }, + { "service", Parameter::PT_INT, nullptr, nullptr, "service id" }, + { "version", Parameter::PT_STRING, nullptr, nullptr, "client version" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + static const Command host_cache_cmds[] = { { "dump", host_cache_dump, host_cache_cmd_params, "dump host cache"}, + { "delete_host", host_cache_delete_host, host_cache_delete_host_params, "delete host from host cache"}, + { "delete_network_proto", host_cache_delete_network_proto, + host_cache_delete_network_proto_params, "delete network protocol from host"}, + { "delete_transport_proto", host_cache_delete_transport_proto, + host_cache_delete_transport_proto_params, "delete transport protocol from host"}, + { "delete_service", host_cache_delete_service, + host_cache_delete_service_params, "delete service from host"}, + { "delete_client", host_cache_delete_client, + host_cache_delete_client_params, "delete client from host"}, { nullptr, nullptr, nullptr, nullptr } }; @@ -97,7 +350,7 @@ bool HostCacheModule::set(const char*, Value& v, SnortConfig*) bool HostCacheModule::end(const char* fqn, int, SnortConfig* sc) { - if ( memcap && !strcmp(fqn, HOST_CACHE_NAME) ) + if ( memcap and !strcmp(fqn, HOST_CACHE_NAME) ) { if ( Snort::is_reloading() ) sc->register_reload_resource_tuner(new HostCacheReloadTuner(memcap)); @@ -159,10 +412,13 @@ void HostCacheModule::log_host_cache(const char* file_name, bool verbose) << lru_data.size() << " trackers" << endl << endl; for ( const auto& elem : lru_data ) { - str = "IP: "; - str += elem.first.ntop(ip_str); - elem.second->stringify(str); - out_stream << str << endl << endl; + if (elem.second->is_visible() == true) + { + str = "IP: "; + str += elem.first.ntop(ip_str); + elem.second->stringify(str); + out_stream << str << endl << endl; + } } out_stream.close(); diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc index 1cb306a68..e04707e11 100644 --- a/src/host_tracker/host_tracker.cc +++ b/src/host_tracker/host_tracker.cc @@ -22,6 +22,8 @@ #include "config.h" #endif +#include + #include "host_cache.h" #include "host_cache_allocator.cc" #include "host_tracker.h" @@ -50,11 +52,21 @@ bool HostTracker::add_network_proto(const uint16_t type) { lock_guard lck(host_tracker_lock); - for ( const auto& proto : network_protos ) - if ( proto == type ) - return false; + for ( auto& proto : network_protos ) + { + if ( proto.first == type ) + { + if (proto.second == true) + return false; + else + { + proto.second = true; + return true; + } + } + } - network_protos.emplace_back(type); + network_protos.emplace_back(type, true); return true; } @@ -62,11 +74,21 @@ bool HostTracker::add_xport_proto(const uint8_t type) { lock_guard lck(host_tracker_lock); - for ( const auto& proto : xport_protos ) - if ( proto == type ) - return false; + for ( auto& proto : xport_protos ) + { + if ( proto.first == type ) + { + if (proto.second == true) + return false; + else + { + proto.second = true; + return true; + } + } + } - xport_protos.emplace_back(type); + xport_protos.emplace_back(type, true); return true; } @@ -254,11 +276,21 @@ bool HostTracker::add_service(Port port, IpProtocol proto, AppId appid, bool inf if (added) *added = true; } + + if ( s.visibility == false ) + { + if ( added ) + *added = true; + s.visibility = true; + num_visible_services++; + } + return true; } } services.emplace_back(port, proto, appid, inferred_appid); + num_visible_services++; if (added) *added = true; @@ -303,7 +335,7 @@ bool HostTracker::add_client_payload(HostClient& hc, AppId payload, size_t max_p return false; } -bool HostTracker::add_service(HostApplication& app, bool* added) +bool HostTracker::add_service(const HostApplication& app, bool* added) { host_tracker_stats.service_adds++; lock_guard lck(host_tracker_lock); @@ -319,11 +351,21 @@ bool HostTracker::add_service(HostApplication& app, bool* added) if (added) *added = true; } + + if (s.visibility == false) + { + if (added) + *added = true; + s.visibility = true; + num_visible_services++; + } + return true; } } services.emplace_back(app.port, app.proto, app.appid, app.inferred_appid); + num_visible_services++; if (added) *added = true; @@ -350,7 +392,7 @@ AppId HostTracker::get_appid(Port port, IpProtocol proto, bool inferred_only, size_t HostTracker::get_service_count() { lock_guard lck(host_tracker_lock); - return services.size(); + return num_visible_services; } HostApplication* HostTracker::find_service_no_lock(Port port, IpProtocol proto, AppId appid) @@ -359,6 +401,8 @@ HostApplication* HostTracker::find_service_no_lock(Port port, IpProtocol proto, { if ( s.port == port and s.proto == proto ) { + if (s.visibility == false) + return nullptr; if ( appid != APP_ID_NONE and s.appid == appid ) return &s; } @@ -392,15 +436,22 @@ HostApplication HostTracker::add_service(Port port, IpProtocol proto, uint32_t l host_tracker_stats.service_finds++; lock_guard lck(host_tracker_lock); + HostApplication *available = nullptr; + for ( auto& s : services ) { if ( s.port == port and s.proto == proto ) { - if ( appid != APP_ID_NONE and s.appid != appid ) + if ( (appid != APP_ID_NONE and s.appid != appid) or !s.visibility ) { s.appid = appid; is_new = true; s.hits = 1; + if ( !s.visibility ) + { + s.visibility = true; + num_visible_services++; + } } else if ( s.last_seen == 0 ) { @@ -411,12 +462,29 @@ HostApplication HostTracker::add_service(Port port, IpProtocol proto, uint32_t l ++s.hits; s.last_seen = lseen; + return s; } + else if ( !available and !s.visibility ) + available = &s; } is_new = true; host_tracker_stats.service_adds++; + num_visible_services++; + if ( available ) + { + available->port = port; + available->proto = proto; + available->appid = appid; + available->hits = 1; + available->last_seen = lseen; + available->inferred_appid = false; + available->user[0] = '\0'; + available->visibility = true; + return *available; + } + services.emplace_back(port, proto, appid, false, 1, lseen); return services.back(); } @@ -449,6 +517,20 @@ void HostTracker::update_service_proto(HostApplication& app, IpProtocol proto) app.proto = proto; } +void HostTracker::update_ha_no_lock(HostApplication& dst, HostApplication& src) +{ + if (dst.appid == APP_ID_NONE) + dst.appid = src.appid; + else + src.appid = dst.appid; + + for (auto& i: src.info) + if (i.visibility == true) + dst.info.emplace_back(i.version, i.vendor); + + dst.hits = src.hits; +} + bool HostTracker::update_service_info(HostApplication& ha, const char* vendor, const char* version, uint16_t max_info) { @@ -459,29 +541,51 @@ bool HostTracker::update_service_info(HostApplication& ha, const char* vendor, { if ( s.port == ha.port and s.proto == ha.proto ) { - if (s.info.size() < max_info) + if ( s.visibility == false ) + return false; + + HostApplicationInfo* available = nullptr; + for ( auto& i : s.info ) { - for (auto& i : s.info) + if (((!version and i.version[0] == '\0') or + (version and !strncmp(version, i.version, INFO_SIZE))) + and ((!vendor and i.vendor[0] == '\0') or + (vendor and !strncmp(vendor, i.vendor, INFO_SIZE)))) { - if (((!version and i.version[0] == '\0') or - (version and !strncmp(version, i.version, INFO_SIZE))) - and ((!vendor and i.vendor[0] == '\0') or - (vendor and !strncmp(vendor, i.vendor, INFO_SIZE)))) - return false; + if (i.visibility == false) + { + i.visibility = true; // rediscover it + update_ha_no_lock(ha, s); + return true; + } + return false; } - s.info.emplace_back(version, vendor); + else if (!available and i.visibility == false) + available = &i; } - // copy these info for the caller - if (ha.appid == APP_ID_NONE) - ha.appid = s.appid; - else - s.appid = ha.appid; + if ( available and (version or vendor) ) + { + if ( version ) + { + strncpy(available->version, version, INFO_SIZE); + available->version[INFO_SIZE-1]='\0'; + } - for (auto& i: s.info) - ha.info.emplace_back(i.version, i.vendor); + if ( vendor ) + { + strncpy(available->vendor, vendor, INFO_SIZE); + available->vendor[INFO_SIZE-1]='\0'; + } - ha.hits = s.hits; + available->visibility = true; + } + else if ( s.info.size() < max_info ) + s.info.emplace_back(version, vendor); + else + return false; + + update_ha_no_lock(ha, s); return true; } } @@ -492,11 +596,13 @@ bool HostTracker::update_service_user(Port port, IpProtocol proto, const char* u { host_tracker_stats.service_finds++; lock_guard lck(host_tracker_lock); - for ( auto& s : services ) { if ( s.port == port and s.proto == proto ) { + if ( s.visibility == false) + return false; + if ( user and strncmp(user, s.user, INFO_SIZE) ) { strncpy(s.user, user, INFO_SIZE); @@ -528,6 +634,120 @@ bool HostTracker::add_tcp_fingerprint(uint32_t fpid) return result.second; } +bool HostTracker::set_visibility(bool v) +{ + std::lock_guard lck(host_tracker_lock); + bool old_visibility = visibility; + + visibility = v; + + if (visibility == false) + { + for (auto& proto : network_protos) + proto.second = false; + + for (auto& proto : xport_protos) + proto.second = false; + + for (auto& s : services) + { + s.visibility = false; + for (auto& info : s.info) + info.visibility = false; + s.user[0] = '\0'; + } + num_visible_services = 0; + + for ( auto& c : clients ) + c.visibility = false; + num_visible_clients = 0; + + tcp_fpids.clear(); + ua_fps.clear(); + } + + return old_visibility; +} + +bool HostTracker::set_network_proto_visibility(uint16_t proto, bool v) +{ + std::lock_guard lck(host_tracker_lock); + for (auto& pp : network_protos) + { + if (pp.first == proto) + { + pp.second = v; + return true; + } + } + return false; +} + +bool HostTracker::set_xproto_visibility(uint8_t proto, bool v) +{ + std::lock_guard lck(host_tracker_lock); + for (auto& pp : xport_protos) + { + if (pp.first == proto) + { + pp.second = v; + return true; + } + } + return false; +} + +bool HostTracker::set_service_visibility(Port port, IpProtocol proto, bool v) +{ + std::lock_guard lck(host_tracker_lock); + for ( auto& s : services ) + { + if ( s.port == port and s.proto == proto ) + { + if ( s.visibility == true and v == false ) + { + assert(num_visible_services > 0); + num_visible_services--; + } + else if ( s.visibility == false and v == true ) + num_visible_services++; + + s.visibility = v; + if ( s.visibility == false ) + { + for ( auto& info : s.info ) + info.visibility = false; + s.user[0] = '\0'; + } + return true; + } + } + return false; +} + +bool HostTracker::set_client_visibility(const HostClient& hc, bool v) +{ + std::lock_guard lck(host_tracker_lock); + bool deleted = false; + for ( auto& c : clients ) + { + if ( c == hc ) + { + if ( c.visibility == true and v == false ) + { + assert(num_visible_clients > 0 ); + num_visible_clients--; + } + else if (c.visibility == false and v == true) + num_visible_clients++; + + c.visibility = v; + deleted = true; + } + } + return deleted; +} + DeviceFingerprint::DeviceFingerprint(uint32_t id, uint32_t type, bool jb, const char* dev) : fpid(id), fp_type(type), jail_broken(jb) { @@ -563,7 +783,7 @@ bool HostTracker::add_ua_fingerprint(uint32_t fpid, uint32_t fp_type, bool jail_ size_t HostTracker::get_client_count() { lock_guard lck(host_tracker_lock); - return clients.size(); + return num_visible_clients; } HostClient::HostClient(AppId clientid, const char *ver, AppId ser) : @@ -580,18 +800,41 @@ HostClient HostTracker::find_or_add_client(AppId id, const char* version, AppId bool& is_new) { lock_guard lck(host_tracker_lock); - - for ( const auto& c : clients ) + HostClient* available = nullptr; + for ( auto& c : clients ) { if ( c.id != APP_ID_NONE and c.id == id and c.service == service and ((c.version[0] == '\0' and !version) or (version and strncmp(c.version, version, INFO_SIZE) == 0)) ) { + if ( c.visibility == false ) + { + is_new = true; + c.visibility = true; + num_visible_clients++; + } + return c; } + else if ( !available and !c.visibility) + available = &c; } is_new = true; + num_visible_clients++; + if ( available ) + { + available->id = id; + available->service = service; + available->visibility = true; + if ( version ) + { + strncpy(available->version, version, INFO_SIZE); + available->version[INFO_SIZE-1] = '\0'; + } + return *available; + } + clients.emplace_back(id, version, service); return clients.back(); } @@ -653,11 +896,15 @@ void HostTracker::stringify(string& str) } } - if ( !services.empty() ) + if ( num_visible_services > 0 ) { - str += "\nservices size: " + to_string(services.size()); + str += "\nservices size: " + to_string(num_visible_services); + for ( const auto& s : services ) { + if (s.visibility == false) + continue; + str += "\n port: " + to_string(s.port) + ", proto: " + to_string((uint8_t) s.proto); if ( s.appid != APP_ID_NONE ) @@ -670,6 +917,9 @@ void HostTracker::stringify(string& str) if ( !s.info.empty() ) for ( const auto& i : s.info ) { + if (i.visibility == false) + continue; + if ( i.vendor[0] != '\0' ) str += ", vendor: " + string(i.vendor); if ( i.version[0] != '\0' ) @@ -677,21 +927,28 @@ void HostTracker::stringify(string& str) } auto total_payloads = s.payloads.size(); - if ( total_payloads ) + if ( total_payloads > 0 ) { str += ", payload"; str += (total_payloads > 1) ? "s: " : ": "; for ( const auto& pld : s.payloads ) + { str += to_string(pld) + (--total_payloads ? ", " : ""); + } } + if ( *s.user ) + str += ", user: " + string(s.user); } } - if ( !clients.empty() ) + if ( num_visible_clients > 0 ) { - str += "\nclients size: " + to_string(clients.size()); + str += "\nclients size: " + to_string(num_visible_clients); for ( const auto& c : clients ) { + if (c.visibility == false) + continue; + str += "\n id: " + to_string(c.id) + ", service: " + to_string(c.service); if ( c.version[0] != '\0' ) @@ -708,23 +965,33 @@ void HostTracker::stringify(string& str) } } - auto total = network_protos.size(); - if ( total ) + if ( any_of(network_protos.begin(), network_protos.end(), + [] (const NetProto_t& proto) { return proto.second; }) ) { str += "\nnetwork proto: "; + auto total = network_protos.size(); while ( total-- ) - str += to_string(network_protos[total]) + (total? ", " : ""); + { + const auto& proto = network_protos[total]; + if ( proto.second == true ) + str += to_string(proto.first) + (total? ", " : ""); + } } - total = xport_protos.size(); - if ( total ) + if ( any_of(xport_protos.begin(), xport_protos.end(), + [] (const XProto_t& proto) { return proto.second; }) ) { str += "\ntransport proto: "; + auto total = xport_protos.size(); while ( total-- ) - str += to_string(xport_protos[total]) + (total? ", " : ""); + { + const auto& proto = xport_protos[total]; + if ( proto.second == true ) + str += to_string(proto.first) + (total? ", " : ""); + } } - total = tcp_fpids.size(); + auto total = tcp_fpids.size(); if ( total ) { str += "\ntcp fingerprint: "; diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h index 34961f514..f85aebb83 100644 --- a/src/host_tracker/host_tracker.h +++ b/src/host_tracker/host_tracker.h @@ -75,9 +75,13 @@ struct HostApplicationInfo HostApplicationInfo(const char *ver, const char *ven); char vendor[INFO_SIZE] = { 0 }; char version[INFO_SIZE] = { 0 }; + bool visibility = true; + + friend class HostTracker; }; typedef HostCacheAllocIp HostAppInfoAllocator; +typedef AppId Payload_t; struct HostApplication { @@ -97,6 +101,7 @@ struct HostApplication last_seen = ha.last_seen; info = ha.info; payloads = ha.payloads; + visibility = ha.visibility; return *this; } @@ -109,7 +114,12 @@ struct HostApplication char user[INFO_SIZE] = { 0 }; std::vector info; - std::vector> payloads; + std::vector> payloads; + + friend class HostTracker; + +private: + bool visibility = true; }; struct HostClient @@ -119,7 +129,17 @@ struct HostClient AppId id; char version[INFO_SIZE] = { 0 }; AppId service; - std::vector> payloads; + std::vector> payloads; + + bool operator==(const HostClient& c) const + { + return id == c.id and service == c.service; + } + + friend class HostTracker; + +private: + bool visibility = true; }; struct DeviceFingerprint @@ -151,6 +171,10 @@ typedef HostCacheAllocIp HostDeviceFpAllocator; class SO_PUBLIC HostTracker { public: + + typedef std::pair NetProto_t; + typedef std::pair XProto_t; + HostTracker() : hops(-1) { last_seen = nat_count_start = (uint32_t) packet_time(); @@ -171,16 +195,24 @@ public: return last_event; } - std::vector> get_network_protos() + std::vector get_network_protos() { + std::vector out_protos; std::lock_guard lck(host_tracker_lock); - return network_protos; + for (const auto& proto : network_protos) + if ( proto.second ) + out_protos.emplace_back(proto.first); + return out_protos; } - std::vector> get_xport_protos() + std::vector get_xport_protos() { + std::vector out_protos; std::lock_guard lck(host_tracker_lock); - return xport_protos; + for (const auto& proto : xport_protos) + if ( proto.second ) + out_protos.emplace_back(proto.first); + return out_protos; } void set_host_type(HostType rht) @@ -245,7 +277,7 @@ public: // appid detected from one flow to another flow such as BitTorrent. bool add_service(Port, IpProtocol, AppId appid = APP_ID_NONE, bool inferred_appid = false, bool* added = nullptr); - bool add_service(HostApplication&, bool* added = nullptr); + bool add_service(const HostApplication&, bool* added = nullptr); void clear_service(HostApplication&); void update_service_port(HostApplication&, Port); void update_service_proto(HostApplication&, IpProtocol); @@ -315,14 +347,29 @@ public: return ++nat_count; } + bool set_visibility(bool v = true); + + bool is_visible() const + { + std::lock_guard lck(host_tracker_lock); + return visibility; + } + + // the control delete commands do not actually remove objects from + // the host tracker, but just mark them as invisible, until rediscovered. + bool set_network_proto_visibility(uint16_t proto, bool v = true); + bool set_xproto_visibility(uint8_t proto, bool v = true); + bool set_service_visibility(Port, IpProtocol, bool v = true); + bool set_client_visibility(const HostClient&, bool v = true); + private: mutable std::mutex host_tracker_lock; // ensure that updates to a shared object are safe uint8_t hops; // hops from the snort inspector, e.g., zero for ARP uint32_t last_seen; // the last time this host was seen uint32_t last_event; // the last time an event was generated std::list macs; // list guarantees iterator validity on insertion - std::vector> network_protos; - std::vector> xport_protos; + std::vector> network_protos; + std::vector> xport_protos; std::vector services; std::vector clients; std::set, HostCacheAllocIp> tcp_fpids; @@ -335,6 +382,11 @@ private: uint32_t nat_count = 0; uint32_t nat_count_start; // the time nat counting start for this host + bool visibility = true; + + uint32_t num_visible_services = 0; + uint32_t num_visible_clients = 0; + // Hide / delete the constructor from the outside world. We don't want to // have zombie host trackers, i.e. host tracker objects that live outside // the host cache. @@ -353,6 +405,7 @@ private: // lock is actually obtained bool add_payload_no_lock(const AppId, HostApplication*, size_t); HostApplication* find_service_no_lock(Port, IpProtocol, AppId); + void update_ha_no_lock(HostApplication& dst, HostApplication& src); // ... and some unit tests. See Utest.h and UtestMacros.h in cpputest. friend class TEST_host_tracker_add_find_service_test_Test; diff --git a/src/host_tracker/test/host_cache_module_test.cc b/src/host_tracker/test/host_cache_module_test.cc index 261e67df4..567452cdc 100644 --- a/src/host_tracker/test/host_cache_module_test.cc +++ b/src/host_tracker/test/host_cache_module_test.cc @@ -70,7 +70,10 @@ void SnortConfig::register_reload_resource_tuner(ReloadResourceTuner* rrt) { del extern "C" { +typedef ptrdiff_t lua_Integer; + const char* luaL_optlstring(lua_State*, int, const char*, size_t*) { return nullptr; } +int luaL_optinteger(lua_State*, int, lua_Integer) { return 0; } } void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } diff --git a/src/host_tracker/test/host_tracker_test.cc b/src/host_tracker/test/host_tracker_test.cc index 1e7dd10a5..9e108b646 100644 --- a/src/host_tracker/test/host_tracker_test.cc +++ b/src/host_tracker/test/host_tracker_test.cc @@ -134,6 +134,62 @@ TEST(host_tracker, stringify) "\n port: 443, proto: 6, appid: 1122"); } +TEST(host_tracker, rediscover_host) +{ + test_time = 1562198400; // this time will be updated and should not be seen in stringify + HostTracker ht; + + ht.add_service(80, IpProtocol::TCP, 676, true); + ht.add_service(443, IpProtocol::TCP, 1122); + CHECK(ht.get_service_count() == 2); + + bool is_new; + ht.find_or_add_client(1, "one", 100, is_new); + ht.find_or_add_client(2, "two", 200, is_new); + CHECK(ht.get_client_count() == 2); + + ht.set_visibility(false); + CHECK(ht.get_service_count() == 0); + + // rediscover the host, no services and clients should be visible + ht.set_visibility(true); + CHECK(ht.get_service_count() == 0); + CHECK(ht.get_client_count() == 0); + + // rediscover a service, that and only that should be visible + ht.add_service(443, IpProtocol::TCP, 1122); + CHECK(ht.get_service_count() == 1); + + // change the appid of existing service + HostApplication ha(443, IpProtocol::TCP, 1133, false); + bool added; + ht.add_service(ha, &added); + CHECK(added); + + // rediscover service using a different add service function + HostApplication ha2(80, IpProtocol::TCP, 676, false); + bool ret = ht.add_service(ha2, &added); + CHECK(ret); + CHECK(added); + + // and a client + ht.find_or_add_client(2, "one", 200, is_new); + CHECK(ht.get_client_count() == 1); + + string host_tracker_string; + ht.stringify(host_tracker_string); + + string expected; + expected += "\n type: Host, ttl: 0, hops: 255, time: 2019-07-04 00:00:00"; + expected += "\nservices size: 2"; + expected += "\n port: 80, proto: 6, appid: 676, inferred"; + expected += "\n port: 443, proto: 6, appid: 1133"; + expected += "\nclients size: 1"; + expected += "\n id: 2, service: 200, version: one"; + + STRCMP_EQUAL(expected.c_str(), host_tracker_string.c_str()); +} + int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/network_inspectors/rna/rna_pnd.cc b/src/network_inspectors/rna/rna_pnd.cc index 8bb41ed8c..4f43af042 100644 --- a/src/network_inspectors/rna/rna_pnd.cc +++ b/src/network_inspectors/rna/rna_pnd.cc @@ -140,6 +140,13 @@ void RnaPnd::discover_network(const Packet* p, uint8_t ttl) auto ht = host_cache.find_else_create(*src_ip, &new_host); + // If it's a new host, it's automatically visible, so we don't do anything. + // If it's not a new host, we're rediscovering it, so make it visible. + // Also if it was not new (we had it in the cache) and it went from + // not visible to visible, then it's as good as new. + if (!new_host and !ht->set_visibility(true)) + new_host = true; + uint32_t last_seen = ht->get_last_seen(); if ( !new_host ) ht->update_last_seen(); // this should be done always and foremost @@ -173,7 +180,7 @@ void RnaPnd::discover_network(const Packet* p, uint8_t ttl) } } - if ( ht->get_host_type() == HOST_TYPE_HOST and p->is_tcp() ) + if ( p->is_tcp() and ht->get_host_type() == HOST_TYPE_HOST ) discover_host_types_ttl(ht, p, ttl, last_seen, src_ip_ptr, src_mac); uint16_t ptype = rna_get_eth(p);