From: Steve Chew (stechew) Date: Tue, 22 Sep 2020 03:47:14 +0000 (+0000) Subject: Merge pull request #2484 in SNORT/snort3 from ~SHASLAD/snort3:netflow_cache to master X-Git-Tag: 3.0.3-1~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e837f08e0d0b60816b51f231d2c740633c30bc51;p=thirdparty%2Fsnort3.git Merge pull request #2484 in SNORT/snort3 from ~SHASLAD/snort3:netflow_cache to master Squashed commit of the following: commit 405d47d61009943346d438ba86788ba44ebded7b Author: Shashi Lad Date: Tue Sep 15 00:21:35 2020 -0400 netflow: cache support and more v5 decoding --- diff --git a/src/service_inspectors/netflow/CMakeLists.txt b/src/service_inspectors/netflow/CMakeLists.txt index fe9cb90f4..1017d4057 100644 --- a/src/service_inspectors/netflow/CMakeLists.txt +++ b/src/service_inspectors/netflow/CMakeLists.txt @@ -1,9 +1,9 @@ set ( FILE_LIST - netflow.cc - netflow.h + netflow_headers.h netflow_module.cc netflow_module.h + netflow.cc ) if (STATIC_INSPECTORS) diff --git a/src/service_inspectors/netflow/netflow.cc b/src/service_inspectors/netflow/netflow.cc index 0d9db1565..aa0c7b695 100644 --- a/src/service_inspectors/netflow/netflow.cc +++ b/src/service_inspectors/netflow/netflow.cc @@ -23,19 +23,58 @@ #include "config.h" #endif -#include "netflow.h" +#include "netflow_headers.h" #include "netflow_module.h" -#include "host_tracker/host_cache.h" +#include +#include +#include +#include +#include + +#include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" +#include "sfip/sf_ip.h" +#include "utils/util.h" using namespace snort; -using namespace std; THREAD_LOCAL NetflowStats netflow_stats; THREAD_LOCAL ProfileStats netflow_perf_stats; +// Used to create hash of key for indexing into cache. +struct NetflowHash +{ + size_t operator()(const snort::SfIp& ip) const + { + const uint64_t* ip64 = (const uint64_t*) ip.get_ip6_ptr(); + return std::hash() (ip64[0]) ^ + std::hash() (ip64[1]); + } +}; + +// compare struct to use with ip sort +struct IpCompare +{ + bool operator()(const snort::SfIp& a, const snort::SfIp& b) + { return a.less_than(b); } +}; + +// ----------------------------------------------------------------------------- +// static variables +// ----------------------------------------------------------------------------- + +// Used to avoid creating multiple events for the same initiator IP. +// Cache can be thread local since Netflow packets coming from a Netflow +// device will go to the same thread. +typedef std::unordered_map NetflowCache; +static THREAD_LOCAL NetflowCache* netflow_cache = nullptr; + +// cache required to dump the output +static NetflowCache* dump_cache = nullptr; + + // ----------------------------------------------------------------------------- // static functions // ----------------------------------------------------------------------------- @@ -99,6 +138,44 @@ static bool decode_netflow_v5(const unsigned char* data, uint16_t size) if ( first_packet > MAX_TIME or last_packet > MAX_TIME or first_packet > last_packet ) return false; + NetflowSessionRecord record = {}; + + // Invalid source IP address provided + if ( record.initiator_ip.set(&precord->flow_src_addr, AF_INET) != SFIP_SUCCESS ) + return false; + + if ( record.responder_ip.set(&precord->flow_dst_addr, AF_INET) != SFIP_SUCCESS ) + return false; + + if ( record.next_hop_ip.set(&precord->next_hop_addr, AF_INET) != SFIP_SUCCESS ) + return false; + + record.initiator_port = ntohs(precord->src_port); + record.responder_port = ntohs(precord->dst_port); + record.proto = precord->flow_protocol; + record.first_pkt_second = first_packet; + record.last_pkt_second = last_packet; + record.initiator_pkts = ntohl(precord->pkt_count); + record.responder_pkts = 0; + record.initiator_bytes = ntohl(precord->bytes_sent); + record.responder_bytes = 0; + record.tcp_flags = precord->tcp_flags; + record.nf_src_tos = precord->tos; + record.nf_dst_tos = precord->tos; + record.nf_snmp_in = ntohs(precord->snmp_if_in); + record.nf_snmp_out = ntohs(precord->snmp_if_out); + record.nf_src_as = (uint32_t)ntohs(precord->src_as); + record.nf_dst_as = (uint32_t)ntohs(precord->dst_as); + record.nf_src_mask = precord->src_mask; + record.nf_dst_mask = precord->dst_mask; + + // insert record + auto result = netflow_cache->emplace(record.initiator_ip, record); + + // new unique record + if ( result.second ) + ++netflow_stats.unique_flows; + } return true; } @@ -138,19 +215,182 @@ static bool validate_netflow(const Packet* p) return retval; } -// ----------------------------------------------------------------------------- -// non-static functions -// ----------------------------------------------------------------------------- +//------------------------------------------------------------------------- +// inspector stuff +//------------------------------------------------------------------------- + +class NetflowInspector : public snort::Inspector +{ +public: + NetflowInspector(const NetflowConfig*); + ~NetflowInspector() override; + + void tinit() override; + void tterm() override; + + void eval(snort::Packet*) override; + +private: + const NetflowConfig *config; + + bool log_netflow_cache(); + void stringify(std::ofstream&); +}; + +void NetflowInspector::stringify(std::ofstream& file_stream) +{ + std::vector keys; + keys.reserve(dump_cache->size()); + + for (const auto& elem : *dump_cache) + keys.push_back(elem.first); + + std::sort(keys.begin(),keys.end(), IpCompare()); + + std::string str; + SfIpString ip_str; + uint32_t i = 0; + + auto& cache = *dump_cache; + + for (auto elem : keys) + { + str = "Netflow Record #"; + str += std::to_string(++i); + str += "\n"; + + str += " Initiator IP (Port): "; + str += elem.ntop(ip_str); + str += " (" + std::to_string(cache[elem].initiator_port) + ")"; + + str += " -> Responder IP (Port): "; + str += cache[elem].responder_ip.ntop(ip_str); + str += " (" + std::to_string(cache[elem].responder_port) + ")"; + str += "\n"; + + str += " Protocol: "; + str += std::to_string(cache[elem].proto); + + str += " Packets: "; + str += std::to_string(cache[elem].initiator_pkts); + str += "\n"; + + str += " Source Mask: "; + str += std::to_string(cache[elem].nf_src_mask); + + str += " Destination Mask: "; + str += std::to_string(cache[elem].nf_dst_mask); + str += "\n"; + + str += " Next Hop IP: "; + str += cache[elem].next_hop_ip.ntop(ip_str); + str += "\n"; + + str += "------\n"; + file_stream << str << std::endl; + + str.clear(); + + } + return; +} + +bool NetflowInspector::log_netflow_cache() +{ + const char* file_name = config->dump_file; + + // Prevent damaging any existing file, intentionally or not + struct stat file_stat; + if ( stat(file_name, &file_stat) == 0 ) + { + LogMessage("File %s already exists!\n", file_name); + return false; + } + + // open file for writing. + std::ofstream dump_file_stream(file_name); + if ( !dump_file_stream ) + { + LogMessage("Error opening %s for dumping netflow cache\n", file_name); + return false; + } + + // print netflow cache dump + stringify(dump_file_stream); + + dump_file_stream.close(); + + LogMessage("Dumped netflow cache to %s\n", file_name); + + return true; +} + +NetflowInspector::NetflowInspector(const NetflowConfig* pc) +{ + config = pc; + + if ( config->dump_file ) + { + // create dump cache + if ( ! dump_cache ) + dump_cache = new NetflowCache; + } +} + +NetflowInspector::~NetflowInspector() +{ + // config and cache removal + if ( config ) + { + if ( config->dump_file ) + { + // log the cache and delete it + if ( dump_cache ) + { + // making sure we only dump if cache is non-zero + if ( dump_cache->size() != 0 ) + log_netflow_cache(); + delete dump_cache; + dump_cache = nullptr; + } + snort_free((void*)config->dump_file); + } + + delete config; + config = nullptr; + } +} void NetflowInspector::eval(Packet* p) { // precondition - what we registered for assert((p->is_udp() and p->dsize and p->data)); + assert(netflow_cache); if ( ! validate_netflow(p) ) ++netflow_stats.invalid_netflow_pkts; } +void NetflowInspector::tinit() +{ + if ( !netflow_cache ) + netflow_cache = new NetflowCache; +} + +void NetflowInspector::tterm() +{ + if ( config->dump_file and dump_cache ) + { + static std::mutex stats_mutex; + std::lock_guard lock(stats_mutex); + { + // insert each cache + dump_cache->insert(netflow_cache->begin(), netflow_cache->end()); + } + } + delete netflow_cache; +} + //------------------------------------------------------------------------- // api stuff //------------------------------------------------------------------------- @@ -162,7 +402,10 @@ static void netflow_mod_dtor(Module* m) { delete m; } static Inspector* netflow_ctor(Module* m) -{ return new NetflowInspector((NetflowModule*)m); } +{ + NetflowModule *mod = (NetflowModule*)m; + return new NetflowInspector(mod->get_data()); +} static void netflow_dtor(Inspector* p) { delete p; } @@ -204,3 +447,4 @@ const BaseApi* sin_netflow[] = &netflow_api.base, nullptr }; + diff --git a/src/service_inspectors/netflow/netflow.h b/src/service_inspectors/netflow/netflow_headers.h similarity index 86% rename from src/service_inspectors/netflow/netflow.h rename to src/service_inspectors/netflow/netflow_headers.h index 29021fce0..0299e6023 100644 --- a/src/service_inspectors/netflow/netflow.h +++ b/src/service_inspectors/netflow/netflow_headers.h @@ -19,22 +19,41 @@ // netflow.h author Ron Dempster // Shashikant Lad -#ifndef NETFLOW_H -#define NETFLOW_H +#ifndef NETFLOW_HEADERS_H +#define NETFLOW_HEADERS_H #include "flow/flow.h" -namespace snort -{ -struct Packet; -} - -class NetflowModule; - #define NETFLOW_MIN_COUNT 1 #define NETFLOW_MAX_COUNT 256 #define MAX_TIME 2145916799 +struct NetflowSessionRecord +{ + snort::SfIp initiator_ip; + snort::SfIp responder_ip; + snort::SfIp next_hop_ip; + uint8_t proto; + uint16_t initiator_port; + uint16_t responder_port; + uint32_t first_pkt_second; + uint32_t last_pkt_second; + uint64_t initiator_pkts; + uint64_t responder_pkts; + uint64_t initiator_bytes; + uint64_t responder_bytes; + uint16_t tcp_flags; + + uint32_t nf_src_as; + uint32_t nf_dst_as; + uint16_t nf_snmp_in; + uint16_t nf_snmp_out; + uint8_t nf_src_tos; + uint8_t nf_dst_tos; + uint8_t nf_src_mask; + uint8_t nf_dst_mask; +}; + struct Netflow5Hdr { uint16_t version; // Netflow export format version number @@ -82,11 +101,4 @@ struct Netflow9Hdr uint32_t source_id; // A 32-bit value that identifies the Exporter Observation Domain }; -class NetflowInspector : public snort::Inspector -{ -public: - NetflowInspector(NetflowModule*) {} - void eval(snort::Packet*) override; -}; - #endif diff --git a/src/service_inspectors/netflow/netflow_module.cc b/src/service_inspectors/netflow/netflow_module.cc index 2ca8ea959..54c62db17 100644 --- a/src/service_inspectors/netflow/netflow_module.cc +++ b/src/service_inspectors/netflow/netflow_module.cc @@ -24,6 +24,8 @@ #include "netflow_module.h" +#include "utils/util.h" + using namespace snort; // ----------------------------------------------------------------------------- @@ -32,6 +34,8 @@ using namespace snort; static const Parameter netflow_params[] = { + { "dump_file", Parameter::PT_STRING, nullptr, nullptr, + "file name to dump netflow cache on shutdown; won't dump by default" }, { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -42,6 +46,7 @@ static const PegInfo netflow_pegs[] = { CountType::SUM, "version_5", "count of netflow version 5 packets received" }, { CountType::SUM, "version_9", "count of netflow version 9 packets received" }, { CountType::SUM, "invalid_netflow_pkts", "count of invalid netflow packets" }, + { CountType::SUM, "unique_flows", "count of unique netflow flows" }, { CountType::END, nullptr, nullptr}, }; @@ -50,7 +55,43 @@ static const PegInfo netflow_pegs[] = //------------------------------------------------------------------------- NetflowModule::NetflowModule() : Module(NETFLOW_NAME, NETFLOW_HELP, netflow_params) -{ } +{ + conf = nullptr; +} + +NetflowModule::~NetflowModule() +{ + delete conf; +} + +NetflowConfig* NetflowModule::get_data() +{ + NetflowConfig* tmp = conf; + conf = nullptr; + return tmp; +} + +bool NetflowModule::begin(const char*, int, SnortConfig*) +{ + assert(!conf); + conf = new NetflowConfig(); + return true; +} + +bool NetflowModule::set(const char*, Value& v, SnortConfig*) +{ + + if ( v.is("dump_file") ) + { + if ( conf->dump_file ) + snort_free((void*)conf->dump_file); + conf->dump_file = snort_strdup(v.get_string()); + } + else + return false; + + return true; +} PegCount* NetflowModule::get_counts() const { return (PegCount*)&netflow_stats; } diff --git a/src/service_inspectors/netflow/netflow_module.h b/src/service_inspectors/netflow/netflow_module.h index f3ee8dbeb..611001740 100644 --- a/src/service_inspectors/netflow/netflow_module.h +++ b/src/service_inspectors/netflow/netflow_module.h @@ -23,6 +23,7 @@ #define NETFLOW_MODULE_H #include "framework/module.h" +#include "utils/util.h" #define NETFLOW_NAME "netflow" #define NETFLOW_HELP "netflow inspection" @@ -32,6 +33,12 @@ namespace snort struct SnortConfig; } +struct NetflowConfig +{ + NetflowConfig() { dump_file = nullptr; } + const char* dump_file; +}; + struct NetflowStats { PegCount packets; @@ -39,6 +46,7 @@ struct NetflowStats PegCount version_5; PegCount version_9; PegCount invalid_netflow_pkts; + PegCount unique_flows; }; extern THREAD_LOCAL NetflowStats netflow_stats; @@ -48,19 +56,25 @@ class NetflowModule : public snort::Module { public: NetflowModule(); + ~NetflowModule() override; - bool set(const char*, snort::Value&, snort::SnortConfig*) override - {return false; } + bool set(const char*, snort::Value&, snort::SnortConfig*) override; + bool begin(const char*, int, snort::SnortConfig*) override; const PegInfo* get_pegs() const override; PegCount* get_counts() const override; snort::ProfileStats* get_profile() const override; + NetflowConfig* get_data(); Usage get_usage() const override { return INSPECT; } bool is_bindable() const override { return true; } + +private: + NetflowConfig* conf; + }; #endif