From: Masud Hasan (mashasan) Date: Tue, 7 Jun 2022 04:04:32 +0000 (+0000) Subject: Pull request #3440: Netflow host/service discovery X-Git-Tag: 3.1.32.0~16 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=19c519b4b6b517d048d3ccc39ca20951a8930457;p=thirdparty%2Fsnort3.git Pull request #3440: Netflow host/service discovery Merge in SNORT/snort3 from ~MMATIRKO/snort3:netflow_disco to master Squashed commit of the following: commit 60339cfeb1a5142a114415a1f451c752bb614297 Author: Michael Matirko Date: Wed May 11 16:11:33 2022 -0400 netflow: implement RNA integration for host/service discovery --- diff --git a/src/network_inspectors/rna/rna_pnd.cc b/src/network_inspectors/rna/rna_pnd.cc index 3fe45744d..c17eb935f 100644 --- a/src/network_inspectors/rna/rna_pnd.cc +++ b/src/network_inspectors/rna/rna_pnd.cc @@ -182,20 +182,86 @@ bool RnaPnd::analyze_netflow(snort::DataEvent& event) if ( !p ) return false; - const auto& src_ip = p->ptrs.ip_api.get_src(); - const auto& src_ip_ptr = (const struct in6_addr*) src_ip->get_ip6_ptr(); - const auto& src_mac = layer::get_eth_layer(p)->ether_src; NetflowEvent* nfe = static_cast(&event); - const NetflowSessionRecord* nf_record = nfe->get_record(); - // process host and service log events - UNUSED(src_ip_ptr); - UNUSED(src_mac); - UNUSED(nf_record); + if (nfe->get_create_host()) + analyze_netflow_host(nfe); + + if (nfe->get_create_service()) + analyze_netflow_service(nfe); return true; } +void RnaPnd::analyze_netflow_host(NetflowEvent* nfe) +{ + const Packet* p = nfe->get_packet(); + if ( !p ) + return; + + bool new_host = false; + const auto& src_ip = nfe->get_record()->initiator_ip; + const auto& src_ip_ptr = (const struct in6_addr*) src_ip.get_ip6_ptr(); + + auto ht = find_or_create_host_tracker(src_ip, new_host); + + if ( !new_host ) + ht->update_last_seen(); + + const uint8_t src_mac[6] = {0}; + + if ( new_host ) + logger.log(RNA_EVENT_NEW, NEW_HOST, p, &ht, src_ip_ptr, src_mac); + + uint16_t ptype = rna_get_eth(p); + if ( ptype > to_utype(ProtocolId::ETHERTYPE_MINIMUM) ) + { + if ( ht->add_network_proto(ptype) ) + logger.log(RNA_EVENT_NEW, NEW_NET_PROTOCOL, p, &ht, ptype, src_mac, src_ip_ptr, + packet_time()); + } + + ptype = to_utype(p->get_ip_proto_next()); + if ( ht->add_xport_proto(ptype) ) + logger.log(RNA_EVENT_NEW, NEW_XPORT_PROTOCOL, p, &ht, ptype, src_mac, src_ip_ptr, + packet_time()); + + if ( !new_host ) + generate_change_host_update(&ht, p, &src_ip, src_mac, packet_time()); +} + +void RnaPnd::analyze_netflow_service(NetflowEvent* nfe) +{ + + const Packet* p = nfe->get_packet(); + if ( !p ) + return; + + bool new_host = false; + const auto& src_ip = nfe->get_record()->initiator_ip; + const auto& mac_addr = layer::get_eth_layer(p)->ether_src; + uint32_t service = nfe->get_service_id(); + uint16_t port = nfe->get_record()->responder_port; + IpProtocol proto = (IpProtocol) nfe->get_record()->proto; + + auto ht = find_or_create_host_tracker(src_ip, new_host); + ht->update_last_seen(); + + bool is_new = false; + auto ha = ht->add_service(port, proto, (uint32_t) packet_time(), is_new, service); + if ( is_new ) + { + if ( proto == IpProtocol::TCP ) + logger.log(RNA_EVENT_NEW, NEW_TCP_SERVICE, p, &ht, + (const struct in6_addr*) src_ip.get_ip6_ptr(), mac_addr, &ha); + else + logger.log(RNA_EVENT_NEW, NEW_UDP_SERVICE, p, &ht, + (const struct in6_addr*) src_ip.get_ip6_ptr(), mac_addr, &ha); + + ha.hits = 0; + ht->update_service(ha); + } +} void RnaPnd::discover_network_icmp(const Packet* p) { discover_network(p, 0); diff --git a/src/network_inspectors/rna/rna_pnd.h b/src/network_inspectors/rna/rna_pnd.h index 326eadf6d..ed5aca794 100644 --- a/src/network_inspectors/rna/rna_pnd.h +++ b/src/network_inspectors/rna/rna_pnd.h @@ -133,6 +133,8 @@ public: void analyze_smb_fingerprint(snort::DataEvent&); bool analyze_cpe_os_info(snort::DataEvent&); bool analyze_netflow(snort::DataEvent&); + void analyze_netflow_host(snort::NetflowEvent*); + void analyze_netflow_service(snort::NetflowEvent*); // generate change event for all hosts in the ip cache void generate_change_host_update(); diff --git a/src/pub_sub/netflow_event.h b/src/pub_sub/netflow_event.h index dd44ebe6f..f7ce3277a 100644 --- a/src/pub_sub/netflow_event.h +++ b/src/pub_sub/netflow_event.h @@ -31,8 +31,10 @@ namespace snort class NetflowEvent : public DataEvent { public: - NetflowEvent(const snort::Packet* p, const NetflowSessionRecord* rec) - : pkt(p), record(rec) { } + NetflowEvent(const snort::Packet* p, const NetflowSessionRecord* rec, + bool cre_host, bool cre_serv, uint32_t s_id) + : pkt(p), record(rec), create_host(cre_host), + create_service(cre_serv), serviceID(s_id) { } const Packet* get_packet() override { return pkt; } @@ -40,9 +42,21 @@ public: const NetflowSessionRecord* get_record() { return record; } + bool get_create_host() + { return create_host; } + + bool get_create_service() + { return create_service; } + + uint32_t get_service_id() + { return serviceID; } + private: const Packet* pkt; const NetflowSessionRecord* record; + bool create_host; + bool create_service; + uint32_t serviceID = 0; }; } diff --git a/src/service_inspectors/netflow/netflow.cc b/src/service_inspectors/netflow/netflow.cc index 28d9986d7..d2f92a338 100644 --- a/src/service_inspectors/netflow/netflow.cc +++ b/src/service_inspectors/netflow/netflow.cc @@ -30,6 +30,7 @@ #include #include "log/messages.h" +#include "managers/module_manager.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "pub_sub/netflow_event.h" @@ -72,11 +73,13 @@ struct IpCompare { return a.first.less_than(b.first); } }; +static std::unordered_map* udp_srv_map = nullptr; +static std::unordered_map* tcp_srv_map = nullptr; + // ----------------------------------------------------------------------------- // static functions // ----------------------------------------------------------------------------- - -static bool filter_record(const NetflowRules* rules, const int zone, +static const NetflowRule* filter_record(const NetflowRules* rules, const int zone, const SfIp* src, const SfIp* dst) { const SfIp* addr[2] = {src, dst}; @@ -86,7 +89,7 @@ static bool filter_record(const NetflowRules* rules, const int zone, for( auto const& rule : rules->exclude ) { if ( rule.filter_match(address, zone) ) - return false; + return nullptr; } } @@ -95,10 +98,23 @@ static bool filter_record(const NetflowRules* rules, const int zone, for( auto const& rule : rules->include ) { if ( rule.filter_match(address, zone) ) - return true; + return &rule; } } - return false; + return nullptr; +} + +static void publish_service_event(const Packet* p, const NetflowRule* match, NetflowSessionRecord& record) +{ + uint32_t serviceID = 0; + + if (record.proto == (int) ProtocolId::TCP and tcp_srv_map) + serviceID = (*tcp_srv_map)[record.responder_port]; + else if (record.proto == (int) ProtocolId::UDP and udp_srv_map) + serviceID = (*udp_srv_map)[record.responder_port]; + + NetflowEvent event(p, &record, match->create_host, match->create_service, serviceID); + DataBus::publish(NETFLOW_EVENT, event); } static bool version_9_record_update(const unsigned char* data, uint32_t unix_secs, @@ -439,7 +455,8 @@ static bool decode_netflow_v9(const unsigned char* data, uint16_t size, } // filter based on configuration - if ( !filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip) ) + const NetflowRule* match = filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip); + if ( !match ) { records--; continue; @@ -459,9 +476,10 @@ static bool decode_netflow_v9(const unsigned char* data, uint16_t size, if ( !record_status.src_tos ) record.nf_src_tos = record.nf_dst_tos; } - // send create_host and create_service flags too so that rna event handler can log those - NetflowEvent event(p, &record); - DataBus::publish(NETFLOW_EVENT, event); + + if (match->create_service) + publish_service_event(p, match, record); + } if ( netflow_cache->add(record.initiator_ip, record, true) ) @@ -605,7 +623,8 @@ static bool decode_netflow_v5(const unsigned char* data, uint16_t size, if ( record.next_hop_ip.set(&precord->next_hop_addr, AF_INET) != SFIP_SUCCESS ) return false; - if ( !filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip) ) + const NetflowRule* match = filter_record(p_rules, zone, &record.initiator_ip, &record.responder_ip); + if ( !match ) continue; record.initiator_port = ntohs(precord->src_port); @@ -630,9 +649,8 @@ static bool decode_netflow_v5(const unsigned char* data, uint16_t size, if ( netflow_cache->add(record.initiator_ip, record, false) ) ++netflow_stats.unique_flows; - // send create_host and create_service flags too so that rna event handler can log those - NetflowEvent event(p, &record); - DataBus::publish(NETFLOW_EVENT, event); + if (match->create_service) + publish_service_event(p, match, record); } return true; } @@ -671,7 +689,6 @@ static bool validate_netflow(const Packet* p, const NetflowRules* p_rules) return retval; } - //------------------------------------------------------------------------- // inspector stuff //------------------------------------------------------------------------- @@ -870,6 +887,14 @@ NetflowInspector::NetflowInspector(const NetflowConfig* pc) if ( ! dump_cache ) dump_cache = new DumpCache; } + + NetflowModule* mod = (NetflowModule*) ModuleManager::get_module(NETFLOW_NAME); + + if (mod) + { + udp_srv_map = &mod->udp_service_mappings; + tcp_srv_map = &mod->tcp_service_mappings; + } } NetflowInspector::~NetflowInspector() diff --git a/src/service_inspectors/netflow/netflow_headers.h b/src/service_inspectors/netflow/netflow_headers.h index bbc29c8b0..6a18d343b 100644 --- a/src/service_inspectors/netflow/netflow_headers.h +++ b/src/service_inspectors/netflow/netflow_headers.h @@ -52,7 +52,7 @@ enum NetflowFieldTypes : uint16_t NETFLOW_DST_IPV6 = 28, NETFLOW_SRC_MASK_IPV6 = 29, NETFLOW_DST_MASK_IPV6 = 30, - NETFLOW_DST_TOS = 55 + NETFLOW_DST_TOS = 55, }; struct NetflowSessionRecord diff --git a/src/service_inspectors/netflow/netflow_module.cc b/src/service_inspectors/netflow/netflow_module.cc index 06908db3b..e2bcc6d2f 100644 --- a/src/service_inspectors/netflow/netflow_module.cc +++ b/src/service_inspectors/netflow/netflow_module.cc @@ -22,6 +22,11 @@ #include "config.h" #endif +#include +#include +#include +#include + #include "netflow_module.h" #include "utils/util.h" @@ -71,6 +76,9 @@ static const Parameter netflow_params[] = { "template_memcap", Parameter::PT_INT, "0:maxSZ", "0", "maximum memory for template cache in bytes, 0 = unlimited" }, + { "netflow_service_id_path", Parameter::PT_STRING, nullptr, nullptr, + "path to file containing service IDs for NetFlow" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -204,9 +212,44 @@ bool NetflowModule::set(const char*, Value& v, SnortConfig*) { rule_cfg.create_service = v.get_bool(); } + else if ( v.is("netflow_service_id_path") ) + { + parse_service_id_file(v.get_string()); + } return true; } +void NetflowModule::parse_service_id_file(const std::string& serv_id_file_path) +{ + std::string serv_line; + std::ifstream serv_id_file; + serv_id_file.open(serv_id_file_path); + + if ( serv_id_file.is_open() ) + { + while ( std::getline(serv_id_file, serv_line) ) + { + std::stringstream ss(serv_line); + std::vector tokens; + + std::string tmp_str; + + while( std::getline(ss, tmp_str, '\t') ) + tokens.push_back(tmp_str); + + // Format is + uint16_t srv_port = std::stoi(tokens[0]); + std::string proto_str = tokens[1]; + uint16_t id = std::stoi(tokens[2]); + + if ( proto_str == "tcp" ) + tcp_service_mappings[srv_port] = id; + else if ( proto_str == "udp" ) + udp_service_mappings[srv_port] = id; + } + } +} + PegCount* NetflowModule::get_counts() const { return (PegCount*)&netflow_stats; } diff --git a/src/service_inspectors/netflow/netflow_module.h b/src/service_inspectors/netflow/netflow_module.h index 15011d65e..e905ea118 100644 --- a/src/service_inspectors/netflow/netflow_module.h +++ b/src/service_inspectors/netflow/netflow_module.h @@ -156,6 +156,11 @@ public: snort::ProfileStats* get_profile() const override; NetflowConfig* get_data(); + void parse_service_id_file(const std::string& serv_id_file_path); + + std::unordered_map udp_service_mappings; + std::unordered_map tcp_service_mappings; + Usage get_usage() const override { return INSPECT; }