From: Masud Hasan (mashasan) <mashasan@cisco.com>
Date: Wed, 29 Sep 2021 19:39:43 +0000 (+0000)
Subject: Merge pull request #3068 in SNORT/snort3 from ~ARMANDAV/snort3:rna_tls_fingerprinting... 
X-Git-Tag: 3.1.14.0~11
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=42bfeef6c682eb26e51618927ca4947ffcff44c9;p=thirdparty%2Fsnort3.git

Merge pull request #3068 in SNORT/snort3 from ~ARMANDAV/snort3:rna_tls_fingerprinting to master

Squashed commit of the following:

commit 064f3c63f264e14a74acf46a89dbfa7fd8cc5da2
Author: armandav <armandav@cisco.com>
Date:   Mon Sep 20 06:42:33 2021 -0400

    rna: Support CPE new os RNA event
---

diff --git a/src/host_tracker/host_tracker.cc b/src/host_tracker/host_tracker.cc
index 830a76409..492df191e 100644
--- a/src/host_tracker/host_tracker.cc
+++ b/src/host_tracker/host_tracker.cc
@@ -807,6 +807,13 @@ bool HostTracker::add_smb_fingerprint(uint32_t fpid)
     return result.second;
 }
 
+bool HostTracker::add_cpe_os_hash(uint32_t hash)
+{
+    lock_guard<mutex> lck(host_tracker_lock);
+    auto result = cpe_fpids.emplace(hash);
+    return result.second;
+}
+
 bool HostTracker::set_visibility(bool v)
 {
     // get_valid_id may use its own lock, so get this outside our lock
@@ -852,6 +859,7 @@ bool HostTracker::set_visibility(bool v)
         udp_fpids.clear();
         smb_fpids.clear();
         netbios_name.clear();
+        cpe_fpids.clear();
     }
 
     return old_visibility == visibility;
diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h
index 7d0e84012..860eef587 100644
--- a/src/host_tracker/host_tracker.h
+++ b/src/host_tracker/host_tracker.h
@@ -337,6 +337,8 @@ public:
     bool add_udp_fingerprint(uint32_t fpid);
     bool add_smb_fingerprint(uint32_t fpid);
 
+    bool add_cpe_os_hash(uint32_t hash);
+
     //  This should be updated whenever HostTracker data members are changed
     void stringify(std::string& str);
 
@@ -432,6 +434,7 @@ private:
     std::set<uint32_t, std::less<uint32_t>, HostCacheAllocIp<uint32_t>> tcp_fpids;
     std::set<uint32_t, std::less<uint32_t>, HostCacheAllocIp<uint32_t>> udp_fpids;
     std::set<uint32_t, std::less<uint32_t>, HostCacheAllocIp<uint32_t>> smb_fpids;
+    std::set<uint32_t, std::less<uint32_t>, HostCacheAllocIp<uint32_t>> cpe_fpids;
     std::vector<DeviceFingerprint, HostDeviceFpAllocator> ua_fps;
     std::string netbios_name;
 
diff --git a/src/network_inspectors/rna/CMakeLists.txt b/src/network_inspectors/rna/CMakeLists.txt
index df05ffaef..fb2e957d2 100644
--- a/src/network_inspectors/rna/CMakeLists.txt
+++ b/src/network_inspectors/rna/CMakeLists.txt
@@ -8,6 +8,7 @@ set (RNA_INCLUDES
     rna_inspector.h
     rna_logger.h
     rna_name.h
+    rna_cpe_os.h
 )
 
 set ( RNA_SOURCES
diff --git a/src/network_inspectors/rna/rna_app_discovery.cc b/src/network_inspectors/rna/rna_app_discovery.cc
index 7203e9138..3b0cf191d 100644
--- a/src/network_inspectors/rna/rna_app_discovery.cc
+++ b/src/network_inspectors/rna/rna_app_discovery.cc
@@ -388,7 +388,7 @@ void RnaAppDiscovery::discover_user(const Packet* p, DiscoveryFilter& filter, RN
 {
     assert(rna_flow);
     RnaTracker rt = rna_flow->get_tracker(p, filter);
-    if ( !rt or !rt->is_visible() )
+    if ( !rt )
         return;
 
     if ( rt->update_service_user(p->flow->server_port, proto, username,
@@ -404,7 +404,7 @@ void RnaAppDiscovery::discover_netbios_name(const snort::Packet* p, DiscoveryFil
     RNAFlow* rna_flow, RnaLogger& logger, const char* nb_name)
 {
     RnaTracker rt = rna_flow->get_tracker(p, filter);
-    if ( !rt or !rt->is_visible())
+    if ( !rt )
         return;
 
     if ( rt->set_netbios_name(nb_name) )
diff --git a/src/network_inspectors/rna/rna_cpe_os.h b/src/network_inspectors/rna/rna_cpe_os.h
new file mode 100644
index 000000000..68e83e4b7
--- /dev/null
+++ b/src/network_inspectors/rna/rna_cpe_os.h
@@ -0,0 +1,57 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2021-2021 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// rna_cpe_os.h author Arun Prasad Mandava <armandav@cisco.com>
+
+#ifndef RNA_CPE_OS_H
+#define RNA_CPE_OS_H
+
+#define CPE_OS_INFO_EVENT "cpe_os_info_event"
+
+class SO_PUBLIC CpeOsInfoEvent : public snort::DataEvent
+{
+public:
+    CpeOsInfoEvent(const snort::Packet& p) : p(p) { }
+
+    const snort::Packet* get_packet() override
+    {
+         return &p;
+    }
+
+    void add_os(const char *name)
+    {
+        hash ^= std::hash<std::string>{}(name);
+        os_names.emplace_back(name);
+    }
+
+    uint32_t get_hash()
+    {
+        return hash;
+    }
+
+    const std::vector<const char*>* get_os_names()
+    {
+        return &os_names;
+    }
+private:
+    const snort::Packet& p;
+    std::vector<const char*> os_names;
+    uint32_t hash = 0;
+};
+
+#endif
diff --git a/src/network_inspectors/rna/rna_event_handler.cc b/src/network_inspectors/rna/rna_event_handler.cc
index 4892202f5..ec0761d01 100644
--- a/src/network_inspectors/rna/rna_event_handler.cc
+++ b/src/network_inspectors/rna/rna_event_handler.cc
@@ -126,3 +126,10 @@ void RnaFpSMBEventHandler::handle(DataEvent& event, Flow*)
     ++rna_stats.smb;
     pnd.analyze_smb_fingerprint(event);
 }
+
+void RnaCPEOSInfoEventHandler::handle(DataEvent& event, Flow*)
+{
+    Profile profile(rna_perf_stats);
+    ++rna_stats.cpe_os;
+    pnd.analyze_cpe_os_info(event);
+}
diff --git a/src/network_inspectors/rna/rna_event_handler.h b/src/network_inspectors/rna/rna_event_handler.h
index 2c766759d..0dfa386f3 100644
--- a/src/network_inspectors/rna/rna_event_handler.h
+++ b/src/network_inspectors/rna/rna_event_handler.h
@@ -153,4 +153,13 @@ private:
     RnaPnd& pnd;
 };
 
+class RnaCPEOSInfoEventHandler : public snort::DataHandler
+{
+public:
+    RnaCPEOSInfoEventHandler(RnaPnd& nd) : DataHandler(RNA_NAME), pnd(nd) { }
+    void handle(snort::DataEvent&, snort::Flow*) override;
+private:
+    RnaPnd& pnd;
+};
+
 #endif
diff --git a/src/network_inspectors/rna/rna_fingerprint.h b/src/network_inspectors/rna/rna_fingerprint.h
index 36abd9bca..de2c9dae7 100644
--- a/src/network_inspectors/rna/rna_fingerprint.h
+++ b/src/network_inspectors/rna/rna_fingerprint.h
@@ -57,7 +57,8 @@ public:
         FP_TYPE_CLIENT6 = 11,
         FP_TYPE_DHCP6 = 12,
         FP_TYPE_USERAGENT = 13,
-        MAX_FP_TYPES = 14
+        FP_TYPE_CPE = 14,
+        MAX_FP_TYPES = 15
     };
 
     uint32_t fpid = 0;
diff --git a/src/network_inspectors/rna/rna_flow.cc b/src/network_inspectors/rna/rna_flow.cc
index 95d97167c..dde407043 100644
--- a/src/network_inspectors/rna/rna_flow.cc
+++ b/src/network_inspectors/rna/rna_flow.cc
@@ -88,7 +88,7 @@ RnaTracker RNAFlow::get_client(const SfIp& ip)
 
 RnaTracker RNAFlow::get_tracker(const Packet* p, DiscoveryFilter& filter)
 {
-    RnaTracker rt;
+    RnaTracker rt = nullptr;
     if ( p->is_from_server() && filter.is_host_monitored(p, nullptr, nullptr, FlowCheckDirection::DF_SERVER) )
         rt = get_server(p->flow->server_ip);
     else if (p->is_from_client() && filter.is_host_monitored(p, nullptr, nullptr, FlowCheckDirection::DF_CLIENT) )
@@ -96,6 +96,9 @@ RnaTracker RNAFlow::get_tracker(const Packet* p, DiscoveryFilter& filter)
 
     if ( rt && rt->is_visible() )
         rt->update_last_seen();
+    else
+        return nullptr;
+
     return rt;
 }
 
diff --git a/src/network_inspectors/rna/rna_inspector.cc b/src/network_inspectors/rna/rna_inspector.cc
index c0cda3c71..92e1d9322 100644
--- a/src/network_inspectors/rna/rna_inspector.cc
+++ b/src/network_inspectors/rna/rna_inspector.cc
@@ -35,7 +35,7 @@
 #include "protocols/packet.h"
 #include "pub_sub/dhcp_events.h"
 #include "pub_sub/smb_events.h"
-
+#include "rna_cpe_os.h"
 #include "rna_event_handler.h"
 #include "rna_fingerprint_smb.h"
 #include "rna_fingerprint_tcp.h"
@@ -105,6 +105,8 @@ bool RnaInspector::configure(SnortConfig* sc)
     DataBus::subscribe_global( STREAM_TCP_SYN_EVENT, new RnaTcpSynEventHandler(*pnd), sc );
     DataBus::subscribe_global( STREAM_TCP_SYN_ACK_EVENT, new RnaTcpSynAckEventHandler(*pnd), sc );
     DataBus::subscribe_global( STREAM_TCP_MIDSTREAM_EVENT, new RnaTcpMidstreamEventHandler(*pnd), sc );
+    DataBus::subscribe_global( CPE_OS_INFO_EVENT, new RnaCPEOSInfoEventHandler(*pnd), sc );
+
     if (rna_conf && rna_conf->log_when_idle)
         DataBus::subscribe_global( THREAD_IDLE_EVENT, new RnaIdleEventHandler(*pnd), sc );
 
diff --git a/src/network_inspectors/rna/rna_logger.cc b/src/network_inspectors/rna/rna_logger.cc
index aba1bb4e7..3c91b0c8d 100644
--- a/src/network_inspectors/rna/rna_logger.cc
+++ b/src/network_inspectors/rna/rna_logger.cc
@@ -212,12 +212,22 @@ void RnaLogger::log(uint16_t type, uint16_t subtype, const snort::Packet* p, Rna
         nullptr, nullptr, APP_ID_NONE, nullptr, false, lease, netmask, router);
 }
 
+void RnaLogger::log(uint16_t type, uint16_t subtype, const snort::Packet* p, RnaTracker* ht,
+    const struct in6_addr* src_ip, const uint8_t* src_mac, const FpFingerprint* fp,
+    const vector<const char*>* cpeos, uint32_t event_time)
+{
+    log(type, subtype, src_ip, src_mac, ht, p, event_time, 0, nullptr, nullptr, fp,
+        nullptr, nullptr, nullptr, APP_ID_NONE, nullptr, false, 0, 0, nullptr,
+        nullptr, cpeos);
+}
+
 bool RnaLogger::log(uint16_t type, uint16_t subtype, const struct in6_addr* src_ip,
     const uint8_t* src_mac, RnaTracker* ht, const Packet* p, uint32_t event_time,
     uint16_t proto, const HostMac* hm, const HostApplication* ha,
     const FpFingerprint* fp, void* cond_var, const HostClient* hc,
     const char* user, AppId appid, const char* di, bool jb, uint32_t lease,
-    uint32_t netmask, const struct in6_addr* router, const char* nb_name)
+    uint32_t netmask, const struct in6_addr* router, const char* nb_name,
+    const vector<const char*>* cpeos)
 {
     if ( !enabled )
         return false;
@@ -225,7 +235,7 @@ bool RnaLogger::log(uint16_t type, uint16_t subtype, const struct in6_addr* src_
     assert(ht);
 
     RnaLoggerEvent rle(type, subtype, src_mac, ht, hm, proto, cond_var,
-        ha, fp, hc, user, appid, di, jb, lease, netmask, router, p, nb_name);
+        ha, fp, hc, user, appid, di, jb, lease, netmask, router, p, nb_name, cpeos);
     if ( src_ip and (!IN6_IS_ADDR_V4MAPPED(src_ip) or src_ip->s6_addr32[3]) )
         rle.ip = src_ip;
     else
diff --git a/src/network_inspectors/rna/rna_logger.h b/src/network_inspectors/rna/rna_logger.h
index 127c2ea18..fe626185f 100644
--- a/src/network_inspectors/rna/rna_logger.h
+++ b/src/network_inspectors/rna/rna_logger.h
@@ -23,7 +23,7 @@
 #include "events/event.h"
 #include "host_tracker/host_cache.h"
 #include "host_tracker/host_tracker.h"
-
+#include "rna_cpe_os.h"
 #include "rna_flow.h"
 
 namespace snort
@@ -39,10 +39,11 @@ struct RnaLoggerEvent : public Event
         const snort::HostMac* hmp, uint16_t pr, void* cv, const snort::HostApplication* hap,
         const snort::FpFingerprint* fpr, const snort::HostClient* hcp, const char* u,
         int32_t app, const char* di, bool jb, uint32_t ls, uint32_t nm,
-        const struct in6_addr* rtr, const snort::Packet* p, const char* nb_name) : type(t), subtype(st),
+        const struct in6_addr* rtr, const snort::Packet* p, const char* nb_name,
+        const std::vector<const char*>* cpe) : type(t), subtype(st),
         mac(mc), ht(rt), hm(hmp), proto(pr), cond_var(cv), ha(hap), fp(fpr), hc(hcp),
         user(u), appid(app), device_info(di), jail_broken(jb), lease(ls), netmask(nm),
-        router(rtr), pkt(p), netbios_name(nb_name) { }
+        router(rtr), pkt(p), netbios_name(nb_name), cpe_os(cpe) { }
 
     uint32_t event_time = 0;
     uint16_t type;
@@ -65,6 +66,7 @@ struct RnaLoggerEvent : public Event
     const struct in6_addr* router;
     const snort::Packet* pkt;
     const char* netbios_name = nullptr;
+    const std::vector<const char*>* cpe_os = nullptr;
 };
 
 class RnaLogger
@@ -84,6 +86,11 @@ public:
     void log(uint16_t type, uint16_t subtype, const snort::Packet*, RnaTracker*,
         const struct in6_addr*, const char* user, AppId appid, uint32_t event_time);
 
+    // for cpe os info event
+    void log(uint16_t type, uint16_t subtype, const snort::Packet* p, RnaTracker* ht,
+        const struct in6_addr* src_ip, const uint8_t* src_mac, const snort::FpFingerprint* fp,
+        const std::vector<const char*>* cpeos, uint32_t event_time);
+
     // for fingerprint
     void log(uint16_t type, uint16_t subtype, const snort::Packet* p, RnaTracker* ht,
         const struct in6_addr* src_ip, const uint8_t* src_mac, const snort::FpFingerprint* fp,
@@ -120,7 +127,8 @@ public:
         void* cond_var = nullptr, const snort::HostClient* hc = nullptr,
         const char* user = nullptr, AppId appid = APP_ID_NONE, const char* device_info = nullptr,
         bool jail_broken = false, uint32_t lease = 0, uint32_t netmask = 0,
-        const struct in6_addr* router = nullptr, const char* nb_name = nullptr);
+        const struct in6_addr* router = nullptr, const char* nb_name = nullptr,
+        const std::vector<const char*>* cpeos = nullptr);
 
 private:
     const bool enabled;
diff --git a/src/network_inspectors/rna/rna_module.cc b/src/network_inspectors/rna/rna_module.cc
index 4847509f2..711d3f864 100644
--- a/src/network_inspectors/rna/rna_module.cc
+++ b/src/network_inspectors/rna/rna_module.cc
@@ -371,6 +371,7 @@ static const Parameter rna_params[] =
 static const PegInfo rna_pegs[] =
 {
     { CountType::SUM, "appid_change", "count of appid change events received" },
+    { CountType::SUM, "cpe_os", "count of CPE OS events received" },
     { CountType::SUM, "icmp_bidirectional", "count of bidirectional ICMP flows received" },
     { CountType::SUM, "icmp_new", "count of new ICMP flows received" },
     { CountType::SUM, "ip_bidirectional", "count of bidirectional IP received" },
diff --git a/src/network_inspectors/rna/rna_module.h b/src/network_inspectors/rna/rna_module.h
index a8b958878..36c846652 100644
--- a/src/network_inspectors/rna/rna_module.h
+++ b/src/network_inspectors/rna/rna_module.h
@@ -34,6 +34,7 @@
 struct RnaStats
 {
     PegCount appid_change;
+    PegCount cpe_os;
     PegCount icmp_bidirectional;
     PegCount icmp_new;
     PegCount ip_bidirectional;
diff --git a/src/network_inspectors/rna/rna_pnd.cc b/src/network_inspectors/rna/rna_pnd.cc
index e0f1a729f..f6ba28b8c 100644
--- a/src/network_inspectors/rna/rna_pnd.cc
+++ b/src/network_inspectors/rna/rna_pnd.cc
@@ -37,6 +37,7 @@
 #include "protocols/protocol_ids.h"
 
 #include "rna_app_discovery.h"
+#include "rna_cpe_os.h"
 #include "rna_fingerprint_smb.h"
 #include "rna_fingerprint_tcp.h"
 #include "rna_fingerprint_udp.h"
@@ -145,6 +146,36 @@ void RnaPnd::analyze_flow_udp(const Packet* p)
         discover_network_udp(p);
 }
 
+bool RnaPnd::analyze_cpe_os_info(snort::DataEvent& event)
+{
+    const Packet* p = event.get_packet();
+    if ( !p or !p->flow )
+        return false;
+
+    RNAFlow* rna_flow = (RNAFlow*) p->flow->get_flow_data(RNAFlow::inspector_id);
+    if ( !rna_flow )
+        return false;
+
+    RnaTracker rt = rna_flow->get_tracker(p, filter);
+    if ( !rt )
+        return false;
+
+    CpeOsInfoEvent& cpeos_event = static_cast<CpeOsInfoEvent&>(event);
+    if ( !rt->add_cpe_os_hash(cpeos_event.get_hash()) )
+        return false;
+
+    const auto& src_ip = p->ptrs.ip_api.get_src();
+    const auto& src_ip_ptr = (const struct in6_addr*) src_ip->get_ip6_ptr();
+    const auto& src_mac = layer::get_eth_layer(p)->ether_src;
+    rt->update_last_seen();
+    FpFingerprint fp = FpFingerprint();
+    fp.fp_type = FpFingerprint::FpType::FP_TYPE_CPE;
+    logger.log(RNA_EVENT_NEW, NEW_OS, p, &rt, src_ip_ptr, src_mac, &fp,
+        cpeos_event.get_os_names(), packet_time());
+
+    return true;
+}
+
 void RnaPnd::discover_network_icmp(const Packet* p)
 {
     discover_network(p, 0);
@@ -361,7 +392,7 @@ void RnaPnd::analyze_smb_fingerprint(DataEvent& event)
         return;
 
     RnaTracker rt = rna_flow->get_tracker(p, filter);
-    if ( !rt or !rt->is_visible() )
+    if ( !rt )
         return;
 
     const FpSMBDataEvent& fp_smb_data_event = static_cast<FpSMBDataEvent&>(event);
@@ -999,4 +1030,66 @@ TEST_CASE("RNA pnd", "[non-ip]")
     }
 }
 
+TEST_CASE("RNA pnd cpe os", "[cpe-os]")
+{
+    SECTION("Testing new os RNA event for cpe os")
+    {
+        RNAFlow::init();
+        RNAFlow* rna_flow = new RNAFlow();
+        Packet p;
+        Flow flow;
+        p.flow = &flow;
+        p.flow->set_flow_data(rna_flow);
+
+        // Fill packet structure with required information
+        eth::EtherHdr eh;
+        const char mac[6] = { 00, 01, 02, 03, 04, 05 };
+        ip::IP4Hdr h4;
+        h4.ip_src = 0x65010101;
+        h4.ip_dst = 0x65010102;
+        p.ptrs.ip_api.set(&h4);
+        memcpy(eh.ether_src, mac, MAC_SIZE);
+        p.packet_flags = PKT_FROM_CLIENT;
+        p.num_layers = 1;
+        p.layers[0].start = (const uint8_t*) &eh;
+
+        // Setup host tracker and attach it to rna flow as client
+        auto* src_ip = p.ptrs.ip_api.get_src();
+        bool new_host = false;
+
+        RnaPnd pnd(false, "");
+
+        Packet p2;
+        p2.flow = nullptr;
+
+        // Test to hit sanity check ht is null
+        CpeOsInfoEvent* cpeevent = new CpeOsInfoEvent(p2);
+        cpeevent->add_os("CPE OS one");
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == false);
+        delete(cpeevent);
+        cpeevent = new CpeOsInfoEvent(p);
+        cpeevent->add_os("CPE OS one");
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == false);
+
+        // Check new OS information is invoking logging
+        auto ht = host_cache.find_else_create(*src_ip, &new_host);
+        rna_flow->set_client(ht);
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == true);
+
+        // Check duplicate OS information is not invoking logging
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == false);
+        delete(cpeevent);
+
+        // Check second OS information is invoking logging
+        cpeevent = new CpeOsInfoEvent(p);
+        cpeevent->add_os("CPE OS two");
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == true);
+
+        // Again check duplicate OS information is not invoking logging
+        CHECK(pnd.analyze_cpe_os_info(*cpeevent) == false);
+
+        delete(cpeevent);
+        p.flow->free_flow_data(rna_flow);
+    }
+}
 #endif
diff --git a/src/network_inspectors/rna/rna_pnd.h b/src/network_inspectors/rna/rna_pnd.h
index c2c38bf26..2a68b01c0 100644
--- a/src/network_inspectors/rna/rna_pnd.h
+++ b/src/network_inspectors/rna/rna_pnd.h
@@ -130,6 +130,7 @@ public:
     void analyze_dhcp_fingerprint(snort::DataEvent&);
     void add_dhcp_info(snort::DataEvent&);
     void analyze_smb_fingerprint(snort::DataEvent&);
+    bool analyze_cpe_os_info(snort::DataEvent&);
 
     // generate change event for all hosts in the ip cache
     void generate_change_host_update();