From: Masud Hasan (mashasan) Date: Tue, 22 Sep 2020 01:00:39 +0000 (+0000) Subject: Merge pull request #2474 in SNORT/snort3 from ~ARMANDAV/snort3:rna_service to master X-Git-Tag: 3.0.3-1~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8e6492eda7e29adf06cb97545e6f7177c7dd2f48;p=thirdparty%2Fsnort3.git Merge pull request #2474 in SNORT/snort3 from ~ARMANDAV/snort3:rna_service to master Squashed commit of the following: commit 45fe15c3bfa63927ccb6d9cedb486ebae9f5b739 Author: Arun Mandava Date: Mon Sep 21 15:10:43 2020 -0400 rna: Service discovery with multiple vendor and version support --- diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc index a5770c499..1c818a555 100644 --- a/src/host_tracker/host_tracker.cc +++ b/src/host_tracker/host_tracker.cc @@ -22,8 +22,9 @@ #include "config.h" #endif +#include "host_cache.h" +#include "host_cache_allocator.cc" #include "host_tracker.h" - #include "utils/util.h" using namespace snort; @@ -247,6 +248,39 @@ bool HostTracker::add_service(Port port, IpProtocol proto, AppId appid, bool inf return true; } +void HostTracker::clear_service(HostApplication& ha) +{ + lock_guard lck(host_tracker_lock); + ha = {}; +} + +bool HostTracker::add_service(HostApplication& app, bool* added) +{ + host_tracker_stats.service_adds++; + lock_guard lck(host_tracker_lock); + + for ( auto& s : services ) + { + if ( s.port == app.port and s.proto == app.proto ) + { + if ( s.appid != app.appid and app.appid != APP_ID_NONE ) + { + s.appid = app.appid; + s.inferred_appid = app.inferred_appid; + if (added) + *added = true; + } + return true; + } + } + + services.emplace_back(app.port, app.proto, app.appid, app.inferred_appid); + if (added) + *added = true; + + return true; +} + AppId HostTracker::get_appid(Port port, IpProtocol proto, bool inferred_only, bool allow_port_wildcard) { @@ -280,15 +314,21 @@ HostApplication HostTracker::get_service(Port port, IpProtocol proto, uint32_t l { if ( s.port == port and s.proto == proto ) { - if ( s.appid != appid and appid != APP_ID_NONE ) + if ( appid != APP_ID_NONE and s.appid != appid ) { s.appid = appid; is_new = true; + s.hits = 1; } else if ( s.last_seen == 0 ) + { is_new = true; + s.hits = 1; + } + else + ++s.hits; + s.last_seen = lseen; - ++s.hits; return s; } } @@ -310,14 +350,25 @@ void HostTracker::update_service(const HostApplication& ha) { s.hits = ha.hits; s.last_seen = ha.last_seen; - if ( ha.appid > APP_ID_NONE ) - s.appid = ha.appid; return; } } } -bool HostTracker::update_service_info(HostApplication& ha, const char* vendor, const char* version) +void HostTracker::update_service_port(HostApplication& app, Port port) +{ + lock_guard lck(host_tracker_lock); + app.port = port; +} + +void HostTracker::update_service_proto(HostApplication& app, IpProtocol proto) +{ + lock_guard lck(host_tracker_lock); + app.proto = proto; +} + +bool HostTracker::update_service_info(HostApplication& ha, const char* vendor, + const char* version, uint16_t max_info) { host_tracker_stats.service_finds++; lock_guard lck(host_tracker_lock); @@ -326,23 +377,28 @@ bool HostTracker::update_service_info(HostApplication& ha, const char* vendor, c { if ( s.port == ha.port and s.proto == ha.proto ) { - bool changed = false; - if ( vendor and strncmp(s.vendor, vendor, INFO_SIZE) ) + if (s.info.size() < max_info) { - strncpy(s.vendor, vendor, INFO_SIZE); - s.vendor[INFO_SIZE-1] = '\0'; - changed = true; + for (auto& i : s.info) + { + if (((!version and i.version[0] == '\0') or + (version and !strncmp(version, i.version, INFO_SIZE))) + and ((!vendor and i.vendor[0] == '\0') or + (vendor and !strncmp(vendor, i.vendor, INFO_SIZE)))) + return false; + } + s.info.emplace_back(version, vendor); } - if ( version and strncmp(s.version, version, INFO_SIZE) ) - { - strncpy(s.version, version, INFO_SIZE); - s.version[INFO_SIZE-1] = '\0'; - changed = true; - } - if ( !changed ) - return false; - ha.appid = s.appid; // copy these info for the caller + // copy these info for the caller + if (ha.appid == APP_ID_NONE) + ha.appid = s.appid; + else + s.appid = ha.appid; + + for (auto& i: s.info) + ha.info.emplace_back(i.version, i.vendor); + ha.hits = s.hits; return true; } @@ -407,6 +463,16 @@ size_t HostTracker::get_client_count() return clients.size(); } +HostClient::HostClient(AppId clientid, const char *ver, AppId ser) : + id(clientid), service(ser) +{ + if (ver) + { + strncpy(version, ver, INFO_SIZE); + version[INFO_SIZE-1] = '\0'; + } +} + HostClient HostTracker::get_client(AppId id, const char* version, AppId service, bool& is_new) { lock_guard lck(host_tracker_lock); @@ -426,6 +492,20 @@ HostClient HostTracker::get_client(AppId id, const char* version, AppId service, return clients.back(); } +HostApplicationInfo::HostApplicationInfo(const char *ver, const char *ven) +{ + if (ver) + { + strncpy(version, ver, INFO_SIZE); + version[INFO_SIZE-1] = '\0'; + } + if (ven) + { + strncpy(vendor, ven, INFO_SIZE); + vendor[INFO_SIZE-1] = '\0'; + } +} + static inline string to_time_string(uint32_t p_time) { time_t raw_time = (time_t) p_time; @@ -474,10 +554,15 @@ void HostTracker::stringify(string& str) if ( s.inferred_appid ) str += ", inferred"; } - if ( s.vendor[0] != '\0' ) - str += ", vendor: " + string(s.vendor); - if ( s.version[0] != '\0' ) - str += ", version: " + string(s.version); + + if ( !s.info.empty() ) + for ( const auto& i : s.info ) + { + if ( i.vendor[0] != '\0' ) + str += ", vendor: " + string(i.vendor); + if ( i.version[0] != '\0' ) + str += ", version: " + string(i.version); + } } } diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h index 1d7890b73..4302e9d68 100644 --- a/src/host_tracker/host_tracker.h +++ b/src/host_tracker/host_tracker.h @@ -66,35 +66,36 @@ struct HostMac uint32_t last_seen; }; +struct HostApplicationInfo +{ + HostApplicationInfo() = default; + HostApplicationInfo(const char *ver, const char *ven); + char vendor[INFO_SIZE] = { 0 }; + char version[INFO_SIZE] = { 0 }; +}; + +typedef HostCacheAllocIp HostAppInfoAllocator; + struct HostApplication { HostApplication() = default; HostApplication(Port pt, IpProtocol pr, AppId ap, bool in, uint32_t ht = 0, uint32_t ls = 0) : port(pt), proto(pr), appid(ap), inferred_appid(in), hits(ht), last_seen(ls) { } - Port port; + Port port = 0; IpProtocol proto; - AppId appid; - bool inferred_appid; - uint32_t hits; - uint32_t last_seen; - char vendor[INFO_SIZE] = { 0 }; - char version[INFO_SIZE] = { 0 }; + AppId appid = APP_ID_NONE; + bool inferred_appid = false; + uint32_t hits = 0; + uint32_t last_seen = 0; + + std::vector info; }; struct HostClient { HostClient() = default; - HostClient(AppId clientid, const char *ver, AppId ser) : - id(clientid), service(ser) - { - if (ver) - { - strncpy(version, ver, INFO_SIZE); - version[INFO_SIZE-1] = '\0'; - } - } - + HostClient(AppId clientid, const char *ver, AppId ser); AppId id; char version[INFO_SIZE] = { 0 }; AppId service; @@ -208,6 +209,10 @@ public: // appid detected from one flow to another flow such as BitTorrent. bool add_service(Port port, IpProtocol proto, AppId appid = APP_ID_NONE, bool inferred_appid = false, bool* added = nullptr); + bool add_service(HostApplication& app, bool* added = nullptr); + void clear_service(HostApplication& hs); + void update_service_port(HostApplication& app, Port port); + void update_service_proto(HostApplication& app, IpProtocol proto); AppId get_appid(Port port, IpProtocol proto, bool inferred_only = false, bool allow_port_wildcard = false); @@ -216,7 +221,8 @@ public: HostApplication get_service(Port port, IpProtocol proto, uint32_t lseen, bool& is_new, AppId appid = APP_ID_NONE); void update_service(const HostApplication& ha); - bool update_service_info(HostApplication& ha, const char* vendor, const char* version); + bool update_service_info(HostApplication& ha, const char* vendor, const char* version, + uint16_t max_info); void remove_inferred_services(); size_t get_client_count(); diff --git a/src/host_tracker/host_tracker_module.cc b/src/host_tracker/host_tracker_module.cc index d7b18fbe2..0274f910b 100644 --- a/src/host_tracker/host_tracker_module.cc +++ b/src/host_tracker/host_tracker_module.cc @@ -62,13 +62,12 @@ bool HostTrackerModule::set(const char*, Value& v, SnortConfig*) v.get_addr(addr); else if ( v.is("port") ) - app.port = v.get_uint16(); - + host_cache[addr]->update_service_port(app, v.get_uint16()); else if ( v.is("proto") ) { const IpProtocol mask[] = { IpProtocol::IP, IpProtocol::TCP, IpProtocol::UDP }; - app.proto = mask[v.get_uint8()]; + host_cache[addr]->update_service_proto(app, mask[v.get_uint8()]); } else @@ -82,7 +81,6 @@ bool HostTrackerModule::begin(const char* fqn, int idx, SnortConfig*) if ( idx && !strcmp(fqn, "host_tracker") ) { addr.clear(); - app = {}; } return true; } @@ -92,12 +90,14 @@ bool HostTrackerModule::end(const char* fqn, int idx, SnortConfig*) if ( idx && !strcmp(fqn, "host_tracker.services") ) { if ( addr.is_set() ) - host_cache[addr]->add_service(app.port, app.proto); - app = {}; + host_cache[addr]->add_service(app); + + host_cache[addr]->clear_service(app); } else if ( idx && !strcmp(fqn, "host_tracker") && addr.is_set() ) { host_cache[addr]; + host_cache[addr]->clear_service(app); addr.clear(); } diff --git a/src/host_tracker/host_tracker_module.h b/src/host_tracker/host_tracker_module.h index e3d1a3c79..6e9888906 100644 --- a/src/host_tracker/host_tracker_module.h +++ b/src/host_tracker/host_tracker_module.h @@ -31,6 +31,7 @@ #include "framework/module.h" #include "host_tracker/host_cache.h" +#include "host_tracker/host_cache_allocator.cc" #define host_tracker_help \ "configure hosts" diff --git a/src/host_tracker/test/host_cache_allocator_test.cc b/src/host_tracker/test/host_cache_allocator_test.cc index 1497262c2..22614e9fc 100644 --- a/src/host_tracker/test/host_cache_allocator_test.cc +++ b/src/host_tracker/test/host_cache_allocator_test.cc @@ -23,7 +23,7 @@ #endif #include "host_tracker/host_cache.h" -#include "host_tracker/host_cache_allocator.h" +#include "host_tracker/host_cache_allocator.cc" #include @@ -32,6 +32,8 @@ #include #include +HostCacheIp host_cache(100); + using namespace std; using namespace snort; diff --git a/src/network_inspectors/rna/rna_app_discovery.cc b/src/network_inspectors/rna/rna_app_discovery.cc index 0163c8722..6298a78a4 100644 --- a/src/network_inspectors/rna/rna_app_discovery.cc +++ b/src/network_inspectors/rna/rna_app_discovery.cc @@ -71,11 +71,17 @@ void RnaAppDiscovery::process(AppidEvent* appid_event, DiscoveryFilter& filter, appid_session_api.get_app_id(&service, &client, &payload, nullptr, nullptr); if ( appid_change_bits[APPID_SERVICE_BIT] and service > APP_ID_NONE ) - discover_service(p, proto, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), - src_mac, conf, logger, service); + { + if ( p->packet_flags & PKT_FROM_SERVER ) + discover_service(p, proto, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), + src_mac, conf, logger, p->flow->server_port, service); + else if ( p->packet_flags & PKT_FROM_CLIENT ) + discover_service(p, proto, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), + src_mac, conf, logger, p->flow->client_port, service); + } if (appid_change_bits[APPID_CLIENT_BIT] and client > APP_ID_NONE - and service > APP_ID_NONE) + and service > APP_ID_NONE ) { const char* version = appid_session_api.get_client_version(); discover_client(p, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), src_mac, @@ -88,8 +94,12 @@ void RnaAppDiscovery::process(AppidEvent* appid_event, DiscoveryFilter& filter, const char* vendor; const char* version; const AppIdServiceSubtype* subtype; + AppId service, client, payload; + + appid_session_api.get_app_id(&service, &client, &payload, nullptr, nullptr); appid_session_api.get_service_info(vendor, version, subtype); - update_service_info(p, proto, vendor, version, ht, src_ip, src_mac, logger); + update_service_info(p, proto, vendor, version, ht, src_ip, src_mac, logger, conf, + service); } if ( p->is_from_client() and ( appid_change_bits[APPID_HOST_BIT] or @@ -113,7 +123,7 @@ void RnaAppDiscovery::process(AppidEvent* appid_event, DiscoveryFilter& filter, void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTracker& rt, const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, - RnaLogger& logger, AppId service) + RnaLogger& logger, uint16_t port, AppId service) { if ( conf and conf->max_host_services and conf->max_host_services <= rt->get_service_count() ) return; @@ -121,7 +131,7 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra bool is_new = false; // Work on a local copy instead of reference as we release lock during event generations - auto ha = rt->get_service(p->flow->server_port, proto, (uint32_t) packet_time(), is_new); + auto ha = rt->get_service(port, proto, (uint32_t) packet_time(), is_new, service); if ( is_new ) { if ( proto == IpProtocol::TCP ) @@ -130,33 +140,23 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra logger.log(RNA_EVENT_NEW, NEW_UDP_SERVICE, p, &rt, src_ip, src_mac, &ha); ha.hits = 0; // hit count is reset after logs are written - ha.appid = service; rt->update_service(ha); } } void RnaAppDiscovery::update_service_info(const Packet* p, IpProtocol proto, const char* vendor, - const char* version, RnaTracker& rt, const SfIp* ip, const uint8_t* src_mac, RnaLogger& logger) + const char* version, RnaTracker& rt, const SfIp* ip, const uint8_t* src_mac, RnaLogger& logger, + RnaConfig* conf, AppId service) { if ( !vendor and !version ) return; - HostApplication ha(p->flow->server_port, proto, APP_ID_NONE, false); - if ( !rt->update_service_info(ha, vendor, version) ) + HostApplication ha(p->flow->server_port, proto, service, false); + if ( !rt->update_service_info(ha, vendor, version, conf->max_host_service_info) ) return; // Work on a local copy for eventing purpose ha.last_seen = (uint32_t) packet_time(); - if ( vendor ) - { - strncpy(ha.vendor, vendor, INFO_SIZE); - ha.vendor[INFO_SIZE-1] = '\0'; - } - if ( version ) - { - strncpy(ha.version, version, INFO_SIZE); - ha.version[INFO_SIZE-1] = '\0'; - } if ( proto == IpProtocol::TCP ) logger.log(RNA_EVENT_CHANGE, CHANGE_TCP_SERVICE_INFO, p, &rt, diff --git a/src/network_inspectors/rna/rna_app_discovery.h b/src/network_inspectors/rna/rna_app_discovery.h index 6e44f28a5..513e5251d 100644 --- a/src/network_inspectors/rna/rna_app_discovery.h +++ b/src/network_inspectors/rna/rna_app_discovery.h @@ -29,7 +29,7 @@ public: static void discover_service(const snort::Packet* p, IpProtocol proto, RnaTracker& rt, const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, - RnaLogger& logger, AppId service = APP_ID_NONE); + RnaLogger& logger, uint16_t port, AppId service = APP_ID_NONE); static void discover_client(const snort::Packet* p, RnaTracker& rt, const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, @@ -37,7 +37,7 @@ public: private: static void update_service_info(const snort::Packet* p, IpProtocol proto, const char* vendor, const char* version, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac, - RnaLogger& logger); + RnaLogger& logger, RnaConfig* conf, AppId service); static void analyze_user_agent_fingerprint(const snort::Packet* p, const char* host, const char* uagent, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac, RnaLogger& logger); diff --git a/src/network_inspectors/rna/rna_logger.cc b/src/network_inspectors/rna/rna_logger.cc index da8e94266..bd9ada573 100644 --- a/src/network_inspectors/rna/rna_logger.cc +++ b/src/network_inspectors/rna/rna_logger.cc @@ -68,6 +68,23 @@ static inline void rna_logger_message(const RnaLoggerEvent& rle) debug_logf(rna_trace, nullptr, "RNA client log: client %u, service %u\n", rle.hc->id, rle.hc->service); } + if (rle.ha) + { + debug_logf(rna_trace, nullptr, + "RNA Service Info log: appid: %d proto %u, port: %u\n", + rle.ha->appid, (uint32_t)rle.ha->proto, rle.ha->port); + + for (auto& s: rle.ha->info) + { + if (s.vendor[0] != '\0') + debug_logf(rna_trace, nullptr, "RNA Service Info log: vendor: %s\n", + s.vendor); + + if (s.version[0] != '\0') + debug_logf(rna_trace, nullptr, "RNA Service Info log: version: %s\n", + s.version); + } + } } else debug_logf(rna_trace, nullptr, "RNA log: type %u, subtype %u, mac %s\n", diff --git a/src/network_inspectors/rna/rna_pnd.cc b/src/network_inspectors/rna/rna_pnd.cc index 775cd90e8..c99bdaf61 100644 --- a/src/network_inspectors/rna/rna_pnd.cc +++ b/src/network_inspectors/rna/rna_pnd.cc @@ -178,7 +178,8 @@ void RnaPnd::discover_network(const Packet* p, uint8_t ttl) { auto proto = p->get_ip_proto_next(); if ( proto == IpProtocol::TCP or proto == IpProtocol::UDP ) - RnaAppDiscovery::discover_service(p, proto, ht, src_ip_ptr, src_mac, conf, logger); + RnaAppDiscovery::discover_service(p, proto, ht, src_ip_ptr, src_mac, conf, + logger, p->flow->server_port); } if ( !new_host )