Pull request #3440: Netflow host/service discovery

Merge in SNORT/snort3 from ~MMATIRKO/snort3:netflow_disco to master

Squashed commit of the following:

commit 60339cfeb1a5142a114415a1f451c752bb614297
Author: Michael Matirko <mmatirko@cisco.com>
Date:   Wed May 11 16:11:33 2022 -0400

    netflow: implement RNA integration for host/service discovery

diff --git a/src/network_inspectors/rna/rna_pnd.cc b/src/network_inspectors/rna/rna_pnd.cc
index 3fe45744d4f42e3c49c58961c821f26fccefd610..c17eb935ff63394981a2fc2af394cd9b6d7e7c04 100644 (file)
--- a/src/network_inspectors/rna/rna_pnd.cc
+++ b/src/network_inspectors/rna/rna_pnd.cc
@@ -182,20 +182,86 @@ bool RnaPnd::analyze_netflow(snort::DataEvent& event)
     if ( !p )
         return false;
 
-    const auto& src_ip = p->ptrs.ip_api.get_src();
-    const auto& src_ip_ptr = (const struct in6_addr*) src_ip->get_ip6_ptr();
-    const auto& src_mac = layer::get_eth_layer(p)->ether_src;
     NetflowEvent* nfe = static_cast<NetflowEvent*>(&event);
-    const NetflowSessionRecord* nf_record = nfe->get_record();
 
-    // process host and service log events
-    UNUSED(src_ip_ptr);
-    UNUSED(src_mac);
-    UNUSED(nf_record);
+    if (nfe->get_create_host())
+        analyze_netflow_host(nfe);
+
+    if (nfe->get_create_service())
+        analyze_netflow_service(nfe);
 
     return true;
 }
 
+void RnaPnd::analyze_netflow_host(NetflowEvent* nfe)
+{
+    const Packet* p = nfe->get_packet();
+    if ( !p )
+        return;
+
+    bool new_host = false;
+    const auto& src_ip = nfe->get_record()->initiator_ip;
+    const auto& src_ip_ptr = (const struct in6_addr*) src_ip.get_ip6_ptr();
+
+    auto ht = find_or_create_host_tracker(src_ip, new_host);
+
+    if ( !new_host )
+        ht->update_last_seen();
+
+    const uint8_t src_mac[6] = {0};
+
+    if ( new_host )
+        logger.log(RNA_EVENT_NEW, NEW_HOST, p, &ht, src_ip_ptr, src_mac);
+
+    uint16_t ptype = rna_get_eth(p);
+    if ( ptype > to_utype(ProtocolId::ETHERTYPE_MINIMUM) )
+    {
+        if ( ht->add_network_proto(ptype) )
+            logger.log(RNA_EVENT_NEW, NEW_NET_PROTOCOL, p, &ht, ptype, src_mac, src_ip_ptr,
+                packet_time());
+    }
+
+    ptype = to_utype(p->get_ip_proto_next());
+    if ( ht->add_xport_proto(ptype) )
+        logger.log(RNA_EVENT_NEW, NEW_XPORT_PROTOCOL, p, &ht, ptype, src_mac, src_ip_ptr,
+            packet_time());
+
+    if ( !new_host )
+        generate_change_host_update(&ht, p, &src_ip, src_mac, packet_time());
+}
+
+void RnaPnd::analyze_netflow_service(NetflowEvent* nfe)
+{
+
+    const Packet* p = nfe->get_packet();
+    if ( !p )
+        return;
+
+    bool new_host = false;
+    const auto& src_ip = nfe->get_record()->initiator_ip;
+    const auto& mac_addr = layer::get_eth_layer(p)->ether_src;
+    uint32_t service = nfe->get_service_id();
+    uint16_t port = nfe->get_record()->responder_port;
+    IpProtocol proto = (IpProtocol) nfe->get_record()->proto;
+
+    auto ht = find_or_create_host_tracker(src_ip, new_host);
+    ht->update_last_seen();
+
+    bool is_new = false;
+    auto ha = ht->add_service(port, proto, (uint32_t) packet_time(), is_new, service);
+    if ( is_new )
+    {
+        if ( proto == IpProtocol::TCP )
+            logger.log(RNA_EVENT_NEW, NEW_TCP_SERVICE, p, &ht,
+                (const struct in6_addr*) src_ip.get_ip6_ptr(), mac_addr, &ha);
+        else
+            logger.log(RNA_EVENT_NEW, NEW_UDP_SERVICE, p, &ht,
+                (const struct in6_addr*) src_ip.get_ip6_ptr(), mac_addr, &ha);
+
+        ha.hits = 0;
+        ht->update_service(ha);
+    }
+}
 void RnaPnd::discover_network_icmp(const Packet* p)
 {
     discover_network(p, 0);

diff --git a/src/network_inspectors/rna/rna_pnd.h b/src/network_inspectors/rna/rna_pnd.h
index 326eadf6d3ea57bd49ca4eb56b25f2baa2d0e73e..ed5aca794367bf737df8344f1465d512c9a6c2be 100644 (file)
--- a/src/network_inspectors/rna/rna_pnd.h
+++ b/src/network_inspectors/rna/rna_pnd.h
@@ -133,6 +133,8 @@ public:
     void analyze_smb_fingerprint(snort::DataEvent&);
     bool analyze_cpe_os_info(snort::DataEvent&);
     bool analyze_netflow(snort::DataEvent&);
+    void analyze_netflow_host(snort::NetflowEvent*);
+    void analyze_netflow_service(snort::NetflowEvent*);
 
     // generate change event for all hosts in the ip cache
     void generate_change_host_update();

diff --git a/src/pub_sub/netflow_event.h b/src/pub_sub/netflow_event.h
index dd44ebe6f2b57d10fbaabeb2075ee94b08f6e28b..f7ce3277a1a8d11c01d5ad27e44d1d21563c1fa6 100644 (file)
--- a/src/pub_sub/netflow_event.h
+++ b/src/pub_sub/netflow_event.h
@@ -31,8 +31,10 @@ namespace snort
 class NetflowEvent : public DataEvent
 {
 public:
-    NetflowEvent(const snort::Packet* p, const NetflowSessionRecord* rec)
-        : pkt(p), record(rec) { }
+    NetflowEvent(const snort::Packet* p, const NetflowSessionRecord* rec,
+        bool cre_host, bool cre_serv, uint32_t s_id)
+        : pkt(p), record(rec), create_host(cre_host),
+          create_service(cre_serv), serviceID(s_id) { }
 
     const Packet* get_packet() override
     { return pkt; }
@@ -40,9 +42,21 @@ public:
     const NetflowSessionRecord* get_record()
     { return record; }
 
+    bool get_create_host()
+    { return create_host; }
+
+    bool get_create_service()
+    { return create_service; }
+
+    uint32_t get_service_id()
+    { return serviceID; }
+
 private:
     const Packet* pkt;
     const NetflowSessionRecord* record;
+    bool create_host;
+    bool create_service;
+    uint32_t serviceID = 0;
 };
 
 }

diff --git a/src/service_inspectors/netflow/netflow.cc b/src/service_inspectors/netflow/netflow.cc
index 28d9986d73f7525ebc4fde73ed5eb79516a51be6..d2f92a3380b6105a2ca4c8d8403fe53f8fd0522c 100644 (file)
--- a/src/service_inspectors/netflow/netflow.cc
+++ b/src/service_inspectors/netflow/netflow.cc
@@ -30,6 +30,7 @@
 #include <vector>
 
 #include "log/messages.h"
+#include "managers/module_manager.h"
 #include "profiler/profiler.h"
 #include "protocols/packet.h"
 #include "pub_sub/netflow_event.h"
@@ -72,11 +73,13 @@ struct IpCompare
     { return a.first.less_than(b.first); }
 };
 
+static std::unordered_map<int, int>* udp_srv_map = nullptr;
+static std::unordered_map<int, int>* tcp_srv_map = nullptr;
+
 // -----------------------------------------------------------------------------
 // static functions
 // -----------------------------------------------------------------------------
-
-static bool filter_record(const NetflowRules* rules, const int zone,
+static const NetflowRule* filter_record(const NetflowRules* rules, const int zone,
     const SfIp* src, const SfIp* dst)
 {
     const SfIp* addr[2] = {src, dst};
@@ -86,7 +89,7 @@ static bool filter_record(const NetflowRules* rules, const int zone,
         for( auto const& rule : rules->exclude )
         {
             if ( rule.filter_match(address, zone) )
-                return false;
+                return nullptr;
         }
     }
 
@@ -95,10 +98,23 @@ static bool filter_record(const NetflowRules* rules, const int zone,
         for( auto const& rule : rules->include )
         {
             if ( rule.filter_match(address, zone) )
-                return true;
+                return &rule;
         }
     }
-    return false;
+    return nullptr;
+}
+
+static void publish_service_event(const Packet* p, const NetflowRule* match, NetflowSessionRecord& record)
+{
+    uint32_t serviceID = 0;
+
+    if (record.proto == (int) ProtocolId::TCP and tcp_srv_map)
+        serviceID = (*tcp_srv_map)[record.responder_port];
+    else if (record.proto == (int) ProtocolId::UDP and udp_srv_map)
+        serviceID = (*udp_srv_map)[record.responder_port];
+
+    NetflowEvent event(p, &record, match->create_host, match->create_service, serviceID);
+    DataBus::publish(NETFLOW_EVENT, event);
 }
 
 static bool version_9_record_update(const unsigned char* data, uint32_t unix_secs,
@@ -439,7 +455,8 @@ static bool decode_netflow_v9(const unsigned char* data, uint16_t size,
                 }
 
                 // filter based on configuration
-                if ( !filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip) )
+                const NetflowRule* match = filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip);
+                if ( !match )
                 {
                     records--;
                     continue;
@@ -459,9 +476,10 @@ static bool decode_netflow_v9(const unsigned char* data, uint16_t size,
                         if ( !record_status.src_tos )
                             record.nf_src_tos = record.nf_dst_tos;
                     }
-                    // send create_host and create_service flags too so that rna event handler can log those
-                    NetflowEvent event(p, &record);
-                    DataBus::publish(NETFLOW_EVENT, event);
+
+                    if (match->create_service)
+                        publish_service_event(p, match, record);
+
                 }
 
                 if ( netflow_cache->add(record.initiator_ip, record, true) )
@@ -605,7 +623,8 @@ static bool decode_netflow_v5(const unsigned char* data, uint16_t size,
         if ( record.next_hop_ip.set(&precord->next_hop_addr, AF_INET) != SFIP_SUCCESS )
             return false;
 
-        if ( !filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip) )
+        const NetflowRule* match = filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip);
+        if ( !match )
             continue;
 
         record.initiator_port = ntohs(precord->src_port);
@@ -630,9 +649,8 @@ static bool decode_netflow_v5(const unsigned char* data, uint16_t size,
         if ( netflow_cache->add(record.initiator_ip, record, false) )
             ++netflow_stats.unique_flows;
 
-        // send create_host and create_service flags too so that rna event handler can log those
-        NetflowEvent event(p, &record);
-        DataBus::publish(NETFLOW_EVENT, event);
+        if (match->create_service)
+            publish_service_event(p, match, record);
     }
     return true;
 }
@@ -671,7 +689,6 @@ static bool validate_netflow(const Packet* p, const NetflowRules* p_rules)
 
     return retval;
 }
-
 //-------------------------------------------------------------------------
 // inspector stuff
 //-------------------------------------------------------------------------
@@ -870,6 +887,14 @@ NetflowInspector::NetflowInspector(const NetflowConfig* pc)
         if ( ! dump_cache )
             dump_cache = new DumpCache;
     }
+
+    NetflowModule* mod = (NetflowModule*) ModuleManager::get_module(NETFLOW_NAME);
+
+    if (mod)
+    {
+        udp_srv_map = &mod->udp_service_mappings;
+        tcp_srv_map = &mod->tcp_service_mappings;
+    }
 }
 
 NetflowInspector::~NetflowInspector()

diff --git a/src/service_inspectors/netflow/netflow_headers.h b/src/service_inspectors/netflow/netflow_headers.h
index bbc29c8b07e105c248fc8725869afc26aaf20fe1..6a18d343b5ea97197a6406e25d81bcd1e6a7b3b6 100644 (file)
--- a/src/service_inspectors/netflow/netflow_headers.h
+++ b/src/service_inspectors/netflow/netflow_headers.h
@@ -52,7 +52,7 @@ enum NetflowFieldTypes : uint16_t
     NETFLOW_DST_IPV6 = 28,
     NETFLOW_SRC_MASK_IPV6 = 29,
     NETFLOW_DST_MASK_IPV6 = 30,
-    NETFLOW_DST_TOS = 55
+    NETFLOW_DST_TOS = 55,
 };
 
 struct NetflowSessionRecord

diff --git a/src/service_inspectors/netflow/netflow_module.cc b/src/service_inspectors/netflow/netflow_module.cc
index 06908db3bcbfc8e297820ae45fb0a9409ee3606f..e2bcc6d2f8a1b6f1a158933d5c51e41fef119513 100644 (file)
--- a/src/service_inspectors/netflow/netflow_module.cc
+++ b/src/service_inspectors/netflow/netflow_module.cc
@@ -22,6 +22,11 @@
 #include "config.h"
 #endif
 
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+
 #include "netflow_module.h"
 
 #include "utils/util.h"
@@ -71,6 +76,9 @@ static const Parameter netflow_params[] =
     { "template_memcap", Parameter::PT_INT, "0:maxSZ", "0",
       "maximum memory for template cache in bytes, 0 = unlimited" },
 
+    { "netflow_service_id_path", Parameter::PT_STRING, nullptr, nullptr,
+      "path to file containing service IDs for NetFlow" },
+
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
@@ -204,9 +212,44 @@ bool NetflowModule::set(const char*, Value& v, SnortConfig*)
     {
         rule_cfg.create_service = v.get_bool();
     }
+    else if ( v.is("netflow_service_id_path") )
+    {
+        parse_service_id_file(v.get_string());
+    }
     return true;
 }
 
+void NetflowModule::parse_service_id_file(const std::string& serv_id_file_path)
+{
+    std::string serv_line;
+    std::ifstream serv_id_file;
+    serv_id_file.open(serv_id_file_path);
+
+    if ( serv_id_file.is_open() )
+    {
+        while ( std::getline(serv_id_file, serv_line) )
+        {
+            std::stringstream ss(serv_line);
+            std::vector<std::string> tokens;
+
+            std::string tmp_str;
+
+            while( std::getline(ss, tmp_str, '\t') )
+                tokens.push_back(tmp_str);
+
+            // Format is <port> <tcp/udp> <internal ID>
+            uint16_t srv_port = std::stoi(tokens[0]);
+            std::string proto_str = tokens[1];
+            uint16_t id = std::stoi(tokens[2]);
+
+            if ( proto_str == "tcp" )
+                tcp_service_mappings[srv_port] = id;
+            else if ( proto_str == "udp" )
+                udp_service_mappings[srv_port] = id;
+        }
+    }
+}
+
 PegCount* NetflowModule::get_counts() const
 { return (PegCount*)&netflow_stats; }
 

diff --git a/src/service_inspectors/netflow/netflow_module.h b/src/service_inspectors/netflow/netflow_module.h
index 15011d65ea4ea1eef46e67255911a13d3a366366..e905ea118be458fad4b75736649a8d1d0b7e6fa2 100644 (file)
--- a/src/service_inspectors/netflow/netflow_module.h
+++ b/src/service_inspectors/netflow/netflow_module.h
@@ -156,6 +156,11 @@ public:
     snort::ProfileStats* get_profile() const override;
     NetflowConfig* get_data();
 
+    void parse_service_id_file(const std::string& serv_id_file_path);
+
+    std::unordered_map<int, int> udp_service_mappings;
+    std::unordered_map<int, int> tcp_service_mappings;
+
     Usage get_usage() const override
     { return INSPECT; }