From: Masud Hasan (mashasan) Date: Mon, 5 Oct 2020 23:30:21 +0000 (+0000) Subject: Merge pull request #2517 in SNORT/snort3 from ~MMATIRKO/snort3:payload_disco to master X-Git-Tag: 3.0.3-2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=346c68d833dd37a1e17ea49a2aa7d3e34a9dff93;p=thirdparty%2Fsnort3.git Merge pull request #2517 in SNORT/snort3 from ~MMATIRKO/snort3:payload_disco to master Squashed commit of the following: commit e7492a2d30552ee06fd9739e04c3411dbb58fe6f Author: Michael Matirko Date: Thu Sep 24 15:15:57 2020 -0400 rna: add payload discovery logic --- diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc index e17a0a72f..2180b2b0c 100644 --- a/src/host_tracker/host_tracker.cc +++ b/src/host_tracker/host_tracker.cc @@ -85,6 +85,17 @@ bool HostTracker::add_mac(const uint8_t* mac, uint8_t ttl, uint8_t primary) return true; } + +bool HostTracker::add_payload_no_lock(const AppId pld, HostApplication* ha) +{ + for ( const auto& app : ha->payloads ) + if ( app == pld ) + return false; + + ha->payloads.emplace_back(pld); + return true; +} + bool HostTracker::get_hostmac(const uint8_t* mac, HostMac& hm) { if ( !mac or !memcmp(mac, zero_mac, MAC_SIZE) ) @@ -255,7 +266,14 @@ bool HostTracker::add_service(Port port, IpProtocol proto, AppId appid, bool inf void HostTracker::clear_service(HostApplication& ha) { lock_guard lck(host_tracker_lock); - ha = {}; + ha.port = 0; + ha.proto = (IpProtocol) 0; + ha.appid = (AppId) 0; + ha.inferred_appid = false; + ha.hits = 0; + ha.last_seen = 0; + ha.payloads.clear(); + ha.info.clear(); } bool HostTracker::add_service(HostApplication& app, bool* added) @@ -308,7 +326,39 @@ size_t HostTracker::get_service_count() return services.size(); } -HostApplication HostTracker::get_service(Port port, IpProtocol proto, uint32_t lseen, +HostApplication* HostTracker::find_service_no_lock(Port port, IpProtocol proto, AppId appid) +{ + for ( auto& s : services ) + { + if ( s.port == port and s.proto == proto ) + { + if ( appid != APP_ID_NONE and s.appid == appid ) + return &s; + } + } + + return nullptr; +} + +bool HostTracker::add_payload(HostApplication& local_ha, Port port, IpProtocol proto, AppId payload, + AppId service, size_t max_payloads) +{ + // This lock is responsible for find_service and add_payload + lock_guard lck(host_tracker_lock); + + auto ha = find_service_no_lock(port, proto, service); + + if (ha and ha->payloads.size() < max_payloads) + { + bool success = add_payload_no_lock(payload, ha); + local_ha = *ha; + return success; + } + + return false; +} + +HostApplication HostTracker::add_service(Port port, IpProtocol proto, uint32_t lseen, bool& is_new, AppId appid) { host_tracker_stats.service_finds++; @@ -596,6 +646,15 @@ void HostTracker::stringify(string& str) if ( i.version[0] != '\0' ) str += ", version: " + string(i.version); } + + auto total_payloads = s.payloads.size(); + if ( total_payloads ) + { + str += ", payload"; + str += (total_payloads > 1) ? "s: " : ": "; + for ( const auto& pld : s.payloads ) + str += to_string(pld) + (--total_payloads ? ", " : ""); + } } } diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h index d3b175872..b72ac1282 100644 --- a/src/host_tracker/host_tracker.h +++ b/src/host_tracker/host_tracker.h @@ -84,6 +84,21 @@ struct HostApplication HostApplication() = default; HostApplication(Port pt, IpProtocol pr, AppId ap, bool in, uint32_t ht = 0, uint32_t ls = 0) : port(pt), proto(pr), appid(ap), inferred_appid(in), hits(ht), last_seen(ls) { } + HostApplication(const HostApplication& ha): port(ha.port), proto(ha.proto), appid(ha.appid), + inferred_appid(ha.inferred_appid), hits(ha.hits), last_seen(ha.last_seen), info(ha.info), + payloads(ha.payloads) { } + HostApplication& operator=(const HostApplication& ha) + { + port = ha.port; + proto = ha.proto; + appid = ha.appid; + inferred_appid = ha.inferred_appid; + hits = ha.hits; + last_seen = ha.last_seen; + info = ha.info; + payloads = ha.payloads; + return *this; + } Port port = 0; IpProtocol proto; @@ -94,6 +109,7 @@ struct HostApplication char user[INFO_SIZE] = { 0 }; std::vector info; + std::vector> payloads; }; struct HostClient @@ -199,6 +215,10 @@ public: // Returns true if we changed primary (false->true), false otherwise bool make_primary(const uint8_t* mac); + // Returns true if a new payload entry added, false otherwise + bool add_payload(HostApplication&, Port, IpProtocol, const AppId payload, + const AppId service, size_t max_payloads); + // Returns the hostmac pointer with the highest TTL HostMac* get_max_ttl_hostmac(); @@ -220,21 +240,22 @@ public: // Appid may not be identified always. Inferred means dynamic/runtime // appid detected from one flow to another flow such as BitTorrent. - bool add_service(Port port, IpProtocol proto, + bool add_service(Port, IpProtocol, AppId appid = APP_ID_NONE, bool inferred_appid = false, bool* added = nullptr); - bool add_service(HostApplication& app, bool* added = nullptr); - void clear_service(HostApplication& hs); - void update_service_port(HostApplication& app, Port port); - void update_service_proto(HostApplication& app, IpProtocol proto); + bool add_service(HostApplication&, bool* added = nullptr); + void clear_service(HostApplication&); + void update_service_port(HostApplication&, Port); + void update_service_proto(HostApplication&, IpProtocol); - AppId get_appid(Port port, IpProtocol proto, bool inferred_only = false, + AppId get_appid(Port, IpProtocol, bool inferred_only = false, bool allow_port_wildcard = false); size_t get_service_count(); - HostApplication get_service(Port port, IpProtocol proto, uint32_t lseen, bool& is_new, - AppId appid = APP_ID_NONE); - void update_service(const HostApplication& ha); - bool update_service_info(HostApplication& ha, const char* vendor, const char* version, + + HostApplication add_service(Port, IpProtocol, uint32_t, bool&, AppId appid = APP_ID_NONE); + + void update_service(const HostApplication&); + bool update_service_info(HostApplication&, const char* vendor, const char* version, uint16_t max_info); bool update_service_user(Port, IpProtocol, const char* username); void remove_inferred_services(); @@ -323,6 +344,12 @@ private: template friend class LruCacheShared; + // These two do not lock independently; they are used by payload discovery and called + // from add_payload(HostApplication&, Port, IpProtocol, AppId, AppId, size_t); where the + // lock is actually obtained + bool add_payload_no_lock(const AppId, HostApplication*); + HostApplication* find_service_no_lock(Port, IpProtocol, AppId); + // ... and some unit tests. See Utest.h and UtestMacros.h in cpputest. friend class TEST_host_tracker_add_find_service_test_Test; friend class TEST_host_tracker_stringify_Test; diff --git a/src/network_inspectors/rna/rna_app_discovery.cc b/src/network_inspectors/rna/rna_app_discovery.cc index b61ff9542..d985fbc63 100644 --- a/src/network_inspectors/rna/rna_app_discovery.cc +++ b/src/network_inspectors/rna/rna_app_discovery.cc @@ -87,6 +87,13 @@ void RnaAppDiscovery::process(AppidEvent* appid_event, DiscoveryFilter& filter, discover_client(p, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), src_mac, conf, logger, version, client, service); } + + if ( appid_change_bits[APPID_PAYLOAD_BIT] and payload > APP_ID_NONE and + service > APP_ID_NONE) + { + discover_payload(p, proto, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), + src_mac, conf, logger, service, payload); + } } if ( appid_change_bits[APPID_SERVICE_VENDOR_BIT] or appid_change_bits[APPID_VERSION_BIT] ) @@ -142,7 +149,7 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra bool is_new = false; // Work on a local copy instead of reference as we release lock during event generations - auto ha = rt->get_service(port, proto, (uint32_t) packet_time(), is_new, service); + auto ha = rt->add_service(port, proto, (uint32_t) packet_time(), is_new, service); if ( is_new ) { if ( proto == IpProtocol::TCP ) @@ -155,6 +162,30 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra } } +void RnaAppDiscovery::discover_payload(const Packet* p, IpProtocol proto, RnaTracker& rt, + const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, + RnaLogger& logger, AppId service, AppId payload) +{ + uint16_t lookup_port; + size_t max_payloads = 0; + + if ( p->is_from_client() ) + lookup_port = p->flow->client_port; + else + lookup_port = p->flow->server_port; + + if ( conf and conf->max_payloads ) + max_payloads = conf->max_payloads; + + HostApplication local_ha; + bool new_pld = rt->add_payload(local_ha, lookup_port, proto, payload, service, max_payloads); + + if ( new_pld ) + { + logger.log(RNA_EVENT_CHANGE, CHANGE_CLIENT_APP_UPDATE, p, &rt, src_ip, src_mac, &local_ha); + } +} + void RnaAppDiscovery::update_service_info(const Packet* p, IpProtocol proto, const char* vendor, const char* version, RnaTracker& rt, const SfIp* ip, const uint8_t* src_mac, RnaLogger& logger, RnaConfig* conf, AppId service) diff --git a/src/network_inspectors/rna/rna_app_discovery.h b/src/network_inspectors/rna/rna_app_discovery.h index 19e53f47e..75eceacd0 100644 --- a/src/network_inspectors/rna/rna_app_discovery.h +++ b/src/network_inspectors/rna/rna_app_discovery.h @@ -24,12 +24,15 @@ class RnaAppDiscovery { public: - static void process(AppidEvent* appid_event, DiscoveryFilter& filter, - RnaConfig* conf, RnaLogger& logger); + static void process(AppidEvent*, DiscoveryFilter&, RnaConfig*, RnaLogger&); - static void discover_service(const snort::Packet* p, IpProtocol proto, RnaTracker& rt, - const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, - RnaLogger& logger, uint16_t port, AppId service = APP_ID_NONE); + static void discover_service(const snort::Packet*, IpProtocol, RnaTracker&, + const struct in6_addr*, const uint8_t*, RnaConfig*, RnaLogger&, uint16_t, + AppId service = APP_ID_NONE); + + static void discover_payload(const snort::Packet*, IpProtocol, RnaTracker&, + const struct in6_addr*, const uint8_t*, RnaConfig*, RnaLogger&, + AppId service, AppId payload); static void discover_client(const snort::Packet* p, RnaTracker& rt, const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf, @@ -37,13 +40,15 @@ public: static void discover_user(const snort::Packet*, RnaTracker&, const struct in6_addr*, const uint8_t* src_mac, RnaLogger&, const char* username, AppId, IpProtocol); + private: - static void update_service_info(const snort::Packet* p, IpProtocol proto, const char* vendor, - const char* version, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac, - RnaLogger& logger, RnaConfig* conf, AppId service); - static void analyze_user_agent_fingerprint(const snort::Packet* p, const char* host, - const char* uagent, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac, - RnaLogger& logger); + static void update_service_info(const snort::Packet*, IpProtocol, const char* vendor, + const char* version, RnaTracker&, const snort::SfIp*, const uint8_t*, + RnaLogger&, RnaConfig*, AppId service); + + static void analyze_user_agent_fingerprint(const snort::Packet*, const char* host, + const char* uagent, RnaTracker&, const snort::SfIp*, const uint8_t*, + RnaLogger&); }; #endif diff --git a/src/network_inspectors/rna/rna_logger_common.h b/src/network_inspectors/rna/rna_logger_common.h index 3ffc509e4..87e45248b 100644 --- a/src/network_inspectors/rna/rna_logger_common.h +++ b/src/network_inspectors/rna/rna_logger_common.h @@ -39,6 +39,7 @@ #define CHANGE_HOST_UPDATE 15 #define CHANGE_HOST_TYPE 16 #define CHANGE_VLAN_TAG 18 + #define CHANGE_CLIENT_APP_UPDATE 32 #define RUA_EVENT 1004 #define CHANGE_USER_LOGIN 2