Merge pull request #2517 in SNORT/snort3 from ~MMATIRKO/snort3:payload_disco to master

Squashed commit of the following:

commit e7492a2d30552ee06fd9739e04c3411dbb58fe6f
Author: Michael Matirko <mmatirko@cisco.com>
Date:   Thu Sep 24 15:15:57 2020 -0400

    rna: add payload discovery logic

diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc
index e17a0a72ff6e56cd22a7c900f67dd745a2cf2bf8..2180b2b0c052b0a4becea6b12be21c7515e9d16a 100644 (file)
--- a/src/host_tracker/host_tracker.cc
+++ b/src/host_tracker/host_tracker.cc
@@ -85,6 +85,17 @@ bool HostTracker::add_mac(const uint8_t* mac, uint8_t ttl, uint8_t primary)
     return true;
 }
 
+
+bool HostTracker::add_payload_no_lock(const AppId pld, HostApplication* ha)
+{
+    for ( const auto& app : ha->payloads )
+        if ( app == pld )
+            return false;
+
+    ha->payloads.emplace_back(pld);
+    return true;
+}
+
 bool HostTracker::get_hostmac(const uint8_t* mac, HostMac& hm)
 {
     if ( !mac or !memcmp(mac, zero_mac, MAC_SIZE) )
@@ -255,7 +266,14 @@ bool HostTracker::add_service(Port port, IpProtocol proto, AppId appid, bool inf
 void HostTracker::clear_service(HostApplication& ha)
 {
     lock_guard<mutex> lck(host_tracker_lock);
-    ha = {};
+    ha.port = 0;
+    ha.proto = (IpProtocol) 0;
+    ha.appid = (AppId) 0;
+    ha.inferred_appid = false;
+    ha.hits = 0;
+    ha.last_seen = 0;
+    ha.payloads.clear();
+    ha.info.clear();
 }
 
 bool HostTracker::add_service(HostApplication& app, bool* added)
@@ -308,7 +326,39 @@ size_t HostTracker::get_service_count()
     return services.size();
 }
 
-HostApplication HostTracker::get_service(Port port, IpProtocol proto, uint32_t lseen,
+HostApplication* HostTracker::find_service_no_lock(Port port, IpProtocol proto, AppId appid)
+{
+    for ( auto& s : services )
+    {
+        if ( s.port == port and s.proto == proto )
+        {
+            if ( appid != APP_ID_NONE and s.appid == appid )
+                return &s;
+        }
+    }
+
+    return nullptr;
+}
+
+bool HostTracker::add_payload(HostApplication& local_ha, Port port, IpProtocol proto, AppId payload,
+    AppId service, size_t max_payloads)
+{
+    // This lock is responsible for find_service and add_payload
+    lock_guard<mutex> lck(host_tracker_lock);
+
+    auto ha = find_service_no_lock(port, proto, service);
+
+    if (ha and ha->payloads.size() < max_payloads)
+    {
+        bool success = add_payload_no_lock(payload, ha);
+        local_ha = *ha;
+        return success;
+    }
+
+    return false;
+}
+
+HostApplication HostTracker::add_service(Port port, IpProtocol proto, uint32_t lseen,
     bool& is_new, AppId appid)
 {
     host_tracker_stats.service_finds++;
@@ -596,6 +646,15 @@ void HostTracker::stringify(string& str)
                     if ( i.version[0] != '\0' )
                         str += ", version: " + string(i.version);
                 }
+
+            auto total_payloads = s.payloads.size();
+            if ( total_payloads )
+            {
+                str += ", payload";
+                str += (total_payloads > 1) ? "s: " : ": ";
+                for ( const auto& pld : s.payloads )
+                    str += to_string(pld) + (--total_payloads ? ", " : "");
+            }
         }
     }
 

diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h
index d3b175872993dec5164a98b772bf9f77b30acd62..b72ac1282f8cd1c575ccd819769468b9a5025114 100644 (file)
--- a/src/host_tracker/host_tracker.h
+++ b/src/host_tracker/host_tracker.h
@@ -84,6 +84,21 @@ struct HostApplication
     HostApplication() = default;
     HostApplication(Port pt, IpProtocol pr, AppId ap, bool in, uint32_t ht = 0, uint32_t ls = 0) :
         port(pt), proto(pr), appid(ap), inferred_appid(in), hits(ht), last_seen(ls) { }
+    HostApplication(const HostApplication& ha): port(ha.port), proto(ha.proto), appid(ha.appid),
+        inferred_appid(ha.inferred_appid), hits(ha.hits), last_seen(ha.last_seen), info(ha.info),
+        payloads(ha.payloads) { }
+    HostApplication& operator=(const HostApplication& ha)
+    {
+        port = ha.port;
+        proto = ha.proto;
+        appid = ha.appid;
+        inferred_appid = ha.inferred_appid;
+        hits = ha.hits;
+        last_seen = ha.last_seen;
+        info = ha.info;
+        payloads = ha.payloads;
+        return *this;
+    }
 
     Port port = 0;
     IpProtocol proto;
@@ -94,6 +109,7 @@ struct HostApplication
     char user[INFO_SIZE] = { 0 };
 
     std::vector<HostApplicationInfo, HostAppInfoAllocator> info;
+    std::vector<AppId, HostCacheAllocIp<AppId>> payloads;
 };
 
 struct HostClient
@@ -199,6 +215,10 @@ public:
     // Returns true if we changed primary (false->true), false otherwise
     bool make_primary(const uint8_t* mac);
 
+    // Returns true if a new payload entry added, false otherwise
+    bool add_payload(HostApplication&, Port, IpProtocol, const AppId payload,
+        const AppId service, size_t max_payloads);
+
     // Returns the hostmac pointer with the highest TTL
     HostMac* get_max_ttl_hostmac();
 
@@ -220,21 +240,22 @@ public:
 
     // Appid may not be identified always. Inferred means dynamic/runtime
     // appid detected from one flow to another flow such as BitTorrent.
-    bool add_service(Port port, IpProtocol proto,
+    bool add_service(Port, IpProtocol,
         AppId appid = APP_ID_NONE, bool inferred_appid = false, bool* added = nullptr);
-    bool add_service(HostApplication& app, bool* added = nullptr);
-    void clear_service(HostApplication& hs);
-    void update_service_port(HostApplication& app, Port port);
-    void update_service_proto(HostApplication& app, IpProtocol proto);
+    bool add_service(HostApplication&, bool* added = nullptr);
+    void clear_service(HostApplication&);
+    void update_service_port(HostApplication&, Port);
+    void update_service_proto(HostApplication&, IpProtocol);
 
-    AppId get_appid(Port port, IpProtocol proto, bool inferred_only = false,
+    AppId get_appid(Port, IpProtocol, bool inferred_only = false,
         bool allow_port_wildcard = false);
 
     size_t get_service_count();
-    HostApplication get_service(Port port, IpProtocol proto, uint32_t lseen, bool& is_new,
-        AppId appid = APP_ID_NONE);
-    void update_service(const HostApplication& ha);
-    bool update_service_info(HostApplication& ha, const char* vendor, const char* version,
+
+    HostApplication add_service(Port, IpProtocol, uint32_t, bool&, AppId appid = APP_ID_NONE);
+
+    void update_service(const HostApplication&);
+    bool update_service_info(HostApplication&, const char* vendor, const char* version,
         uint16_t max_info);
     bool update_service_user(Port, IpProtocol, const char* username);
     void remove_inferred_services();
@@ -323,6 +344,12 @@ private:
     template<class Key, class Value, class Hash>
     friend class LruCacheShared;
 
+    // These two do not lock independently; they are used by payload discovery and called
+    // from add_payload(HostApplication&, Port, IpProtocol, AppId, AppId, size_t); where the
+    // lock is actually obtained
+    bool add_payload_no_lock(const AppId, HostApplication*);
+    HostApplication* find_service_no_lock(Port, IpProtocol, AppId);
+
     // ... and some unit tests. See Utest.h and UtestMacros.h in cpputest.
     friend class TEST_host_tracker_add_find_service_test_Test;
     friend class TEST_host_tracker_stringify_Test;

diff --git a/src/network_inspectors/rna/rna_app_discovery.cc b/src/network_inspectors/rna/rna_app_discovery.cc
index b61ff9542e3207e6dcfc3ceb6735f55556543e95..d985fbc634b0aac0aa7b6af14bcfac7dcca3b54a 100644 (file)
--- a/src/network_inspectors/rna/rna_app_discovery.cc
+++ b/src/network_inspectors/rna/rna_app_discovery.cc
@@ -87,6 +87,13 @@ void RnaAppDiscovery::process(AppidEvent* appid_event, DiscoveryFilter& filter,
             discover_client(p, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(), src_mac,
                 conf, logger, version, client, service);
         }
+
+        if ( appid_change_bits[APPID_PAYLOAD_BIT] and payload > APP_ID_NONE and
+            service > APP_ID_NONE)
+        {
+             discover_payload(p, proto, ht, (const struct in6_addr*) src_ip->get_ip6_ptr(),
+                 src_mac, conf, logger, service, payload);
+        }
     }
 
     if ( appid_change_bits[APPID_SERVICE_VENDOR_BIT] or appid_change_bits[APPID_VERSION_BIT] )
@@ -142,7 +149,7 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra
     bool is_new = false;
 
     // Work on a local copy instead of reference as we release lock during event generations
-    auto ha = rt->get_service(port, proto, (uint32_t) packet_time(), is_new, service);
+    auto ha = rt->add_service(port, proto, (uint32_t) packet_time(), is_new, service);
     if ( is_new )
     {
         if ( proto == IpProtocol::TCP )
@@ -155,6 +162,30 @@ void RnaAppDiscovery::discover_service(const Packet* p, IpProtocol proto, RnaTra
     }
 }
 
+void RnaAppDiscovery::discover_payload(const Packet* p, IpProtocol proto, RnaTracker& rt,
+    const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf,
+    RnaLogger& logger, AppId service, AppId payload)
+{
+    uint16_t lookup_port;
+    size_t max_payloads = 0;
+
+    if ( p->is_from_client() )
+        lookup_port = p->flow->client_port;
+    else
+        lookup_port = p->flow->server_port;
+
+    if ( conf and conf->max_payloads )
+        max_payloads = conf->max_payloads;
+
+    HostApplication local_ha;
+    bool new_pld = rt->add_payload(local_ha, lookup_port, proto, payload, service, max_payloads);
+
+    if ( new_pld )
+    {
+        logger.log(RNA_EVENT_CHANGE, CHANGE_CLIENT_APP_UPDATE, p, &rt, src_ip, src_mac, &local_ha);
+    }
+}
+
 void RnaAppDiscovery::update_service_info(const Packet* p, IpProtocol proto, const char* vendor,
     const char* version, RnaTracker& rt, const SfIp* ip, const uint8_t* src_mac, RnaLogger& logger,
     RnaConfig* conf, AppId service)

diff --git a/src/network_inspectors/rna/rna_app_discovery.h b/src/network_inspectors/rna/rna_app_discovery.h
index 19e53f47e0e395592f7015ea00ba1cf87cedd3ba..75eceacd0c91b079260cb9887a8fb26e0b8c9dfe 100644 (file)
--- a/src/network_inspectors/rna/rna_app_discovery.h
+++ b/src/network_inspectors/rna/rna_app_discovery.h
@@ -24,12 +24,15 @@
 class RnaAppDiscovery
 {
 public:
-    static void process(AppidEvent* appid_event, DiscoveryFilter& filter,
-        RnaConfig* conf, RnaLogger& logger);
+    static void process(AppidEvent*, DiscoveryFilter&, RnaConfig*, RnaLogger&);
 
-    static void discover_service(const snort::Packet* p, IpProtocol proto, RnaTracker& rt,
-        const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf,
-        RnaLogger& logger, uint16_t port, AppId service = APP_ID_NONE);
+    static void discover_service(const snort::Packet*, IpProtocol, RnaTracker&,
+        const struct in6_addr*, const uint8_t*, RnaConfig*, RnaLogger&, uint16_t,
+        AppId service = APP_ID_NONE);
+
+    static void discover_payload(const snort::Packet*, IpProtocol, RnaTracker&,
+        const struct in6_addr*, const uint8_t*, RnaConfig*, RnaLogger&,
+        AppId service, AppId payload);
 
     static void discover_client(const snort::Packet* p, RnaTracker& rt,
         const struct in6_addr* src_ip, const uint8_t* src_mac, RnaConfig* conf,
@@ -37,13 +40,15 @@ public:
 
     static void discover_user(const snort::Packet*, RnaTracker&, const struct in6_addr*,
         const uint8_t* src_mac, RnaLogger&, const char* username, AppId, IpProtocol);
+
 private:
-    static void update_service_info(const snort::Packet* p, IpProtocol proto, const char* vendor,
-        const char* version, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac,
-        RnaLogger& logger, RnaConfig* conf, AppId service);
-    static void analyze_user_agent_fingerprint(const snort::Packet* p, const char* host,
-        const char* uagent, RnaTracker& rt, const snort::SfIp* ip, const uint8_t* src_mac,
-        RnaLogger& logger);
+    static void update_service_info(const snort::Packet*, IpProtocol, const char* vendor,
+        const char* version, RnaTracker&, const snort::SfIp*, const uint8_t*,
+        RnaLogger&, RnaConfig*, AppId service);
+
+    static void analyze_user_agent_fingerprint(const snort::Packet*, const char* host,
+        const char* uagent, RnaTracker&, const snort::SfIp*, const uint8_t*,
+        RnaLogger&);
 };
 
 #endif

diff --git a/src/network_inspectors/rna/rna_logger_common.h b/src/network_inspectors/rna/rna_logger_common.h
index 3ffc509e42d257fb71796aeb41380c105cf0c42b..87e45248b8d64b6702b298dd217945dece8aed73 100644 (file)
--- a/src/network_inspectors/rna/rna_logger_common.h
+++ b/src/network_inspectors/rna/rna_logger_common.h
@@ -39,6 +39,7 @@
     #define CHANGE_HOST_UPDATE         15
     #define CHANGE_HOST_TYPE           16
     #define CHANGE_VLAN_TAG            18
+    #define CHANGE_CLIENT_APP_UPDATE   32
 
 #define RUA_EVENT         1004
     #define CHANGE_USER_LOGIN    2