From: Rishabh Duggal (riduggal) Date: Thu, 20 Jun 2024 13:02:25 +0000 (+0000) Subject: Pull request #4298: flow: introducing new parameters for ip flow profiling X-Git-Tag: 3.3.1.0~13 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eed0cc30c9a14f8d060fa141195c9b3308c8556d;p=thirdparty%2Fsnort3.git Pull request #4298: flow: introducing new parameters for ip flow profiling Merge in SNORT/snort3 from ~RIDUGGAL/snort3:ip_attributes to master Squashed commit of the following: commit 39655047db64c6ccbe46145df1f985a88a982715 Author: riduggal Date: Wed Apr 24 11:26:16 2024 +0000 perf_monitor: introducing new parameters for ip flow profiling --- diff --git a/src/flow/flow.h b/src/flow/flow.h index 379a29ed3..4fbb5508b 100644 --- a/src/flow/flow.h +++ b/src/flow/flow.h @@ -144,6 +144,7 @@ struct FlowStats uint64_t server_bytes; struct timeval start_time; uint64_t total_flow_latency; + uint64_t total_rule_latency; }; struct LwState diff --git a/src/latency/latency_module.cc b/src/latency/latency_module.cc index b41508501..88ee5484f 100644 --- a/src/latency/latency_module.cc +++ b/src/latency/latency_module.cc @@ -203,10 +203,14 @@ bool LatencyModule::set(const char* fqn, Value& v, SnortConfig* sc) bool LatencyModule::end(const char*, int, SnortConfig* sc) { - PacketLatencyConfig& config = sc->latency->packet_latency; + PacketLatencyConfig& packet_config = sc->latency->packet_latency; + RuleLatencyConfig& rule_config = sc->latency->rule_latency; - if (config.max_time > CLOCK_ZERO) - config.force_enable = true; + if ( packet_config.max_time > CLOCK_ZERO ) + packet_config.force_enable = true; + + if ( rule_config.max_time > CLOCK_ZERO ) + rule_config.force_enable = true; return true; } diff --git a/src/latency/rule_latency.cc b/src/latency/rule_latency.cc index 7ba03f048..3ae84221e 100644 --- a/src/latency/rule_latency.cc +++ b/src/latency/rule_latency.cc @@ -244,6 +244,8 @@ inline bool Impl::pop() { assert(!timers.empty()); const auto& timer = timers.back(); + if ( timer.packet->flow ) + timer.packet->flow->flowstats.total_rule_latency += clock_usecs(TO_USECS(timer.elapsed())); bool timed_out = false; @@ -338,7 +340,7 @@ static inline Impl<>& get_impl() void RuleLatency::push(const detection_option_tree_root_t& root, Packet* p) { - if ( rule_latency::config->enabled() ) + if ( rule_latency::config->force_enabled() ) { if ( rule_latency::get_impl().push(root, p) ) ++latency_stats.rule_tree_enables; @@ -349,7 +351,7 @@ void RuleLatency::push(const detection_option_tree_root_t& root, Packet* p) void RuleLatency::pop() { - if ( rule_latency::config->enabled() ) + if ( rule_latency::config->force_enabled() ) { if ( rule_latency::get_impl().pop() ) ++latency_stats.rule_eval_timeouts; diff --git a/src/latency/rule_latency_config.h b/src/latency/rule_latency_config.h index 85fa90027..a4f04d4d4 100644 --- a/src/latency/rule_latency_config.h +++ b/src/latency/rule_latency_config.h @@ -26,6 +26,7 @@ struct RuleLatencyConfig { hr_duration max_time = 0_ticks; + bool force_enable = false; bool suspend = false; unsigned suspend_threshold = 0; hr_duration max_suspend_time = 0_ticks; @@ -41,6 +42,12 @@ struct RuleLatencyConfig #endif return max_time > 0_ticks; } + + bool force_enabled() const + { + return force_enable; + } + bool allow_reenable() const { return max_suspend_time > 0_ticks; } }; diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 7464a2920..a672dee5e 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -799,16 +799,40 @@ void SnortConfig::set_overlay_trace_config(TraceConfig* tc) overlay_trace_config = tc; } -bool SnortConfig::set_latency_enable() +bool SnortConfig::set_packet_latency(bool is_enabled) const { - if (latency) + if ( latency ) { - latency->packet_latency.force_enable = true; + latency->packet_latency.force_enable = is_enabled; + return is_enabled; + } + return false; +} + +bool SnortConfig::get_packet_latency() const +{ + if ( latency->packet_latency.force_enabled() ) return true; + return false; +} + +bool SnortConfig::set_rule_latency(bool is_enabled) const +{ + if ( latency ) + { + latency->rule_latency.force_enable = is_enabled; + return is_enabled; } return false; } +bool SnortConfig::get_rule_latency() const +{ + if ( latency->rule_latency.force_enabled() ) + return true; + return false; +} + void SnortConfig::set_tunnel_verdicts(const char* args) { char* tmp, * tok; diff --git a/src/main/snort_config.h b/src/main/snort_config.h index 52e713b10..39f89b5b6 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -470,7 +470,10 @@ public: void set_utc(bool); void set_watchdog(uint16_t); void set_watchdog_min_thread_count(uint16_t); - SO_PUBLIC bool set_latency_enable(); + SO_PUBLIC bool set_packet_latency(bool) const; + SO_PUBLIC bool get_packet_latency() const; + SO_PUBLIC bool set_rule_latency(bool) const; + SO_PUBLIC bool get_rule_latency() const; //------------------------------------------------------ // accessor methods diff --git a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc index 2b7973ab4..32c62694b 100644 --- a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc +++ b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc @@ -24,10 +24,14 @@ #include "flow_ip_tracker.h" +#include +#include "flow/stream_flow.h" +#include "framework/pig_pen.h" #include "hash/hash_defs.h" #include "log/messages.h" #include "protocols/packet.h" +#include "perf_monitor.h" #include "perf_pegs.h" using namespace snort; @@ -43,7 +47,8 @@ struct FlowStateKey }; FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_addr, - int* swapped) + int* swapped, const char* appid_name, uint16_t src_port, uint16_t dst_port, + uint8_t ip_protocol, uint64_t flow_latency, uint64_t rule_latency) { FlowStateKey key; FlowStateValue* value = nullptr; @@ -62,7 +67,25 @@ FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_ } value = (FlowStateValue*)ip_map->get_user_data(&key); - if ( !value ) + if ( value ) + { + strncpy(value->appid_name, appid_name, sizeof(value->appid_name) - 1); + value->appid_name[sizeof(value->appid_name) - 1] = '\0'; + if ( *swapped ) + { + value->port_a = dst_port; + value->port_b = src_port; + } + else + { + value->port_a = src_port; + value->port_b = dst_port; + } + value->protocol = ip_protocol; + value->total_flow_latency = flow_latency; + value->total_rule_latency = rule_latency; + } + else { if ( ip_map->insert(&key, nullptr) != HASH_OK ) return nullptr; @@ -128,6 +151,12 @@ FlowIPTracker::FlowIPTracker(PerfConfig* perf) : PerfTracker(perf, TRACKER_NAME) &stats.state_changes[SFS_STATE_TCP_CLOSED]); formatter->register_field("udp_created", (PegCount*) &stats.state_changes[SFS_STATE_UDP_CREATED]); + formatter->register_field("app_id", appid_name); + formatter->register_field("port_a", port_a); + formatter->register_field("port_b", port_b); + formatter->register_field("protocol", protocol); + formatter->register_field("flow_latency", flow_latency); + formatter->register_field("rule_latency", rule_latency); formatter->finalize_fields(); stats.total_packets = stats.total_bytes = 0; @@ -157,6 +186,38 @@ void FlowIPTracker::update(Packet* p) const SfIp* src_addr = p->ptrs.ip_api.get_src(); const SfIp* dst_addr = p->ptrs.ip_api.get_dst(); + char curr_appid_name[40] = {}; + uint16_t src_port = 0; + uint16_t dst_port = 0; + uint8_t ip_protocol = 0; + uint64_t curr_flow_latency = 0; + uint64_t curr_rule_latency = 0; + + PerfMonitor* perf_monitor = (PerfMonitor*)PigPen::get_inspector(PERF_NAME, true); + if ( perf_monitor->get_constraints()->flow_ip_all == true ) + { + if ( p->flow ) + { + src_port = p->ptrs.sp; + dst_port = p->ptrs.dp; + + const AppIdSessionApi* appid_session_api = appid_api.get_appid_session_api(*p->flow); + if ( appid_session_api ) + { + AppId service_id = APP_ID_NONE; + appid_session_api->get_app_id(&service_id, nullptr, nullptr, nullptr, nullptr); + const char* app_name = appid_api.get_application_name(service_id, *p->flow); + if ( app_name ) + { + strncpy(curr_appid_name, app_name, sizeof(curr_appid_name) - 1); + curr_appid_name[sizeof(curr_appid_name) - 1] = '\0'; + } + } + ip_protocol = p->flow->ip_proto; + curr_flow_latency = p->flow->flowstats.total_flow_latency; + curr_rule_latency = p->flow->flowstats.total_rule_latency; + } + } int len = p->pktlen; if (p->ptrs.tcph) @@ -164,7 +225,8 @@ void FlowIPTracker::update(Packet* p) else if (p->ptrs.udph) type = SFS_TYPE_UDP; - FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); + FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped, curr_appid_name, + src_port, dst_port, ip_protocol, curr_flow_latency, curr_rule_latency); if ( !value ) return; @@ -194,6 +256,19 @@ void FlowIPTracker::process(bool) key->ipA.ntop(ip_a, sizeof(ip_a)); key->ipB.ntop(ip_b, sizeof(ip_b)); + + if (cur_stats->appid_name[0] != '\0') + strncpy(appid_name, cur_stats->appid_name, sizeof(appid_name) - 1); + else + strncpy(appid_name, "APPID_NONE", sizeof(appid_name) - 1); + appid_name[sizeof(appid_name) - 1] = '\0'; + + std::snprintf(port_a, sizeof(port_a), "%d", cur_stats->port_a); + std::snprintf(port_b, sizeof(port_b), "%d", cur_stats->port_b); + std::snprintf(protocol, sizeof(protocol), "%d", cur_stats->protocol); + std::snprintf(flow_latency, sizeof(flow_latency), "%lu", cur_stats->total_flow_latency); + std::snprintf(rule_latency, sizeof(rule_latency), "%lu", cur_stats->total_rule_latency); + memcpy(&stats, cur_stats, sizeof(stats)); write(); @@ -203,11 +278,14 @@ void FlowIPTracker::process(bool) reset(); } -int FlowIPTracker::update_state(const SfIp* src_addr, const SfIp* dst_addr, FlowState state) +int FlowIPTracker::update_state(const SfIp* src_addr, const SfIp* dst_addr, + FlowState state, const char* appid_name, uint16_t src_port, uint16_t dst_port, + uint8_t ip_protocol, uint64_t flow_latency, uint64_t rule_latency) { int swapped; - FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); + FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped, appid_name, src_port, dst_port, + ip_protocol, flow_latency, rule_latency); if ( !value ) return 1; diff --git a/src/network_inspectors/perf_monitor/flow_ip_tracker.h b/src/network_inspectors/perf_monitor/flow_ip_tracker.h index c40fbd375..629582ef8 100644 --- a/src/network_inspectors/perf_monitor/flow_ip_tracker.h +++ b/src/network_inspectors/perf_monitor/flow_ip_tracker.h @@ -23,6 +23,7 @@ #include "hash/xhash.h" +#include "network_inspectors/appid/application_ids.h" #include "perf_tracker.h" enum FlowState @@ -51,10 +52,16 @@ struct TrafficStats struct FlowStateValue { - TrafficStats traffic_stats[SFS_TYPE_MAX]; - PegCount total_packets; - PegCount total_bytes; - PegCount state_changes[SFS_STATE_MAX]; + char appid_name[40] = "APPID_NONE"; + uint16_t port_a = 0; + uint16_t port_b = 0; + uint8_t protocol = 0; + TrafficStats traffic_stats[SFS_TYPE_MAX] = {}; + PegCount total_packets = 0; + PegCount total_bytes = 0; + PegCount total_flow_latency = 0; + PegCount total_rule_latency = 0; + PegCount state_changes[SFS_STATE_MAX] = {}; }; class FlowIPTracker : public PerfTracker @@ -67,18 +74,23 @@ public: void reset() override; void update(snort::Packet*) override; void process(bool) override; - int update_state(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, FlowState); + int update_state(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, FlowState, + const char* appid_name, uint16_t src_port, uint16_t dst_port, uint8_t ip_protocol, + uint64_t flow_latency, uint64_t rule_latency); snort::XHash* get_ip_map() { return ip_map; } private: FlowStateValue stats; snort::XHash* ip_map; - char ip_a[41], ip_b[41]; + char ip_a[41], ip_b[41], port_a[8], port_b[8], protocol[8]; + char appid_name[40] = "APPID_NONE", flow_latency[20] = {}, rule_latency[20] = {}; int perf_flags; PerfConfig* perf_conf; size_t memcap; - FlowStateValue* find_stats(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, int* swapped); + FlowStateValue* find_stats(const snort::SfIp* src_addr, const snort::SfIp* dst_addr, + int* swapped, const char* appid_name, uint16_t src_port, uint16_t dst_port, + uint8_t ip_protocol, uint64_t flow_latency, uint64_t rule_latency); void write_stats(); void display_stats(); diff --git a/src/network_inspectors/perf_monitor/perf_module.cc b/src/network_inspectors/perf_monitor/perf_module.cc index b3c46afd7..6f02f7e87 100644 --- a/src/network_inspectors/perf_monitor/perf_module.cc +++ b/src/network_inspectors/perf_monitor/perf_module.cc @@ -68,6 +68,9 @@ static const Parameter s_params[] = { "flow_ip", Parameter::PT_BOOL, nullptr, "false", "enable statistics on host pairs" }, + { "flow_ip_all", Parameter::PT_BOOL, nullptr, "false", + "enable every stat of flow_ip profiling on host pairs" }, + { "packets", Parameter::PT_INT, "0:max32", "10000", "minimum packets to report" }, @@ -115,6 +118,8 @@ private: PerfMonitor* perf_monitor; }; +static bool current_packet_latency, current_rule_latency = false; + static const Parameter flow_ip_profiling_params[] = { { "seconds", Parameter::PT_INT, "1:max32", nullptr, @@ -123,6 +128,9 @@ static const Parameter flow_ip_profiling_params[] = { "packets", Parameter::PT_INT, "0:max32", nullptr, "minimum packets to report" }, + { "flow_ip_all", Parameter::PT_BOOL, nullptr, nullptr, + "enable all flow ip statistics" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -149,13 +157,25 @@ static int enable_flow_ip_profiling(lua_State* L) } auto* new_constraints = new PerfConstraints(true, luaL_optint(L, 1, 0), - luaL_optint(L, 2, 0)); + luaL_optint(L, 2, 0), luaL_opt(L,lua_toboolean, 3, false)); ControlConn* ctrlcon = ControlConn::query_from_lua(L); main_broadcast_command(new PerfMonFlowIPDebug(new_constraints, true, perf_monitor), ctrlcon); - LogMessage("Enabling flow ip profiling with sample interval %d packet count %d\n", - new_constraints->sample_interval, new_constraints->pkt_cnt); + if ( new_constraints->flow_ip_all ) + { + const SnortConfig* sc = SnortConfig::get_conf(); + if ( sc->get_packet_latency() ) + current_packet_latency = true; + if ( sc->get_rule_latency() ) + current_rule_latency = true; + sc->set_packet_latency(true); + sc->set_rule_latency(true); + } + + LogMessage("Enabling flow ip profiling with sample interval %d packet count %d all stats tracking %s\n", + new_constraints->sample_interval, new_constraints->pkt_cnt, + ( new_constraints->flow_ip_all ) ? "enabled" : "disabled" ); return 0; } @@ -184,6 +204,14 @@ static int disable_flow_ip_profiling(lua_State* L) ControlConn* ctrlcon = ControlConn::query_from_lua(L); main_broadcast_command(new PerfMonFlowIPDebug(new_constraints, false, perf_monitor), ctrlcon); + const SnortConfig* sc = SnortConfig::get_conf(); + + if ( !current_packet_latency ) + sc->set_packet_latency(false); + + if ( !current_rule_latency ) + sc->set_rule_latency(false); + LogMessage("Disabling flow ip profiling\n"); return 0; @@ -209,7 +237,7 @@ static int show_flow_ip_profiling(lua_State* L) static const Command perf_module_cmds[] = { { "enable_flow_ip_profiling", enable_flow_ip_profiling, - flow_ip_profiling_params, "enable statistics on host pairs" }, + flow_ip_profiling_params, "enable all statistics on host pairs" }, { "disable_flow_ip_profiling", disable_flow_ip_profiling, nullptr, "disable statistics on host pairs" }, @@ -270,6 +298,11 @@ bool PerfMonModule::set(const char*, Value& v, SnortConfig*) { config->sample_interval = v.get_uint32(); } + else if ( v.is("flow_ip_all") ) + { + if ( v.get_bool() ) + config->flow_ip_all = true; + } else if ( v.is("flow_ip_memcap") ) { config->flowip_memcap = v.get_size(); @@ -331,6 +364,12 @@ bool PerfMonModule::end(const char* fqn, int idx, SnortConfig* sc) if ( idx != 0 && strcmp(fqn, "perf_monitor.modules") == 0 ) return config->modules.back().confirm_parse(); + if ( config->flow_ip_all ) + { + sc->set_packet_latency(true); + sc->set_rule_latency(true); + } + return true; } @@ -341,6 +380,7 @@ PerfConfig* PerfMonModule::get_config() tmp->constraints->flow_ip_enabled = config->perf_flags & PERF_FLOWIP; tmp->constraints->sample_interval = config->sample_interval; tmp->constraints->pkt_cnt = config->pkt_cnt; + tmp->constraints->flow_ip_all = config->flow_ip_all; config = nullptr; return tmp; diff --git a/src/network_inspectors/perf_monitor/perf_module.h b/src/network_inspectors/perf_monitor/perf_module.h index 2625e0384..04d89c584 100644 --- a/src/network_inspectors/perf_monitor/perf_module.h +++ b/src/network_inspectors/perf_monitor/perf_module.h @@ -78,10 +78,11 @@ struct PerfConstraints bool flow_ip_enabled = false; unsigned sample_interval = 0; uint32_t pkt_cnt = 0; + bool flow_ip_all = false; PerfConstraints() = default; - PerfConstraints(bool en, unsigned interval, uint32_t cnt) : - flow_ip_enabled(en), sample_interval(interval), pkt_cnt(cnt) { } + PerfConstraints(bool en, unsigned interval, uint32_t cnt, bool lat) : + flow_ip_enabled(en), sample_interval(interval), pkt_cnt(cnt), flow_ip_all(lat) { } }; struct PerfConfig @@ -92,6 +93,7 @@ struct PerfConfig uint64_t max_file_size = 0; int flow_max_port_to_track = 0; size_t flowip_memcap = 0; + bool flow_ip_all = false; PerfFormat format = PerfFormat::CSV; PerfOutput output = PerfOutput::TO_FILE; std::vector modules; diff --git a/src/network_inspectors/perf_monitor/perf_monitor.cc b/src/network_inspectors/perf_monitor/perf_monitor.cc index 3a7720802..2bf8beffb 100644 --- a/src/network_inspectors/perf_monitor/perf_monitor.cc +++ b/src/network_inspectors/perf_monitor/perf_monitor.cc @@ -30,6 +30,8 @@ #include "perf_monitor.h" +#include +#include "flow/stream_flow.h" #include "framework/data_bus.h" #include "framework/pig_pen.h" #include "hash/hash_defs.h" @@ -114,7 +116,36 @@ public: if ( state == SFS_STATE_MAX ) return; - tracker->update_state(&flow->client_ip, &flow->server_ip, state); + char appid_name[40] = {}; + uint16_t src_port = 0; + uint16_t dst_port = 0; + uint8_t ip_protocol = 0; + uint64_t flow_latency = 0; + uint64_t rule_latency = 0; + + if ( perf_monitor.get_constraints()->flow_ip_all ) + { + const AppIdSessionApi* appid_session_api = appid_api.get_appid_session_api(*flow); + if ( appid_session_api ) + { + AppId service_id = APP_ID_NONE; + appid_session_api->get_app_id(&service_id, nullptr, nullptr, nullptr, nullptr); + const char* app_name = appid_api.get_application_name(service_id, *flow); + if ( app_name ) + { + strncpy(appid_name, app_name, sizeof(appid_name) - 1); + appid_name[sizeof(appid_name) - 1] = '\0'; + } + } + src_port = flow->client_port; + dst_port = flow->server_port; + ip_protocol = flow->ip_proto; + flow_latency = flow->flowstats.total_flow_latency; + rule_latency = flow->flowstats.total_rule_latency; + } + + tracker->update_state(&flow->client_ip, &flow->server_ip, state, appid_name, + src_port, dst_port, ip_protocol, flow_latency, rule_latency); } private: @@ -128,10 +159,10 @@ static const char* to_string(const PerfOutput& po) { switch (po) { - case PerfOutput::TO_CONSOLE: - return "console"; - case PerfOutput::TO_FILE: - return "file"; + case PerfOutput::TO_CONSOLE: + return "console"; + case PerfOutput::TO_FILE: + return "file"; } return ""; @@ -141,14 +172,14 @@ static const char* to_string(const PerfFormat& pf) { switch (pf) { - case PerfFormat::TEXT: - return "text"; - case PerfFormat::CSV: - return "csv"; - case PerfFormat::JSON: - return "json"; - case PerfFormat::MOCK: - return "mock"; + case PerfFormat::TEXT: + return "text"; + case PerfFormat::CSV: + return "csv"; + case PerfFormat::JSON: + return "json"; + case PerfFormat::MOCK: + return "mock"; } return ""; @@ -164,7 +195,10 @@ void PerfMonitor::show(const SnortConfig*) const ConfigLogger::log_value("flow_ports", config->flow_max_port_to_track); if ( ConfigLogger::log_flag("flow_ip", config->perf_flags & PERF_FLOWIP) ) + { ConfigLogger::log_value("flow_ip_memcap", config->flowip_memcap); + ConfigLogger::log_value("flow_ip_all", config->flow_ip_all); + } ConfigLogger::log_value("packets", config->pkt_cnt); ConfigLogger::log_value("seconds", config->sample_interval); @@ -299,7 +333,7 @@ void PerfMonitor::swap_constraints(PerfConstraints* constraints) PerfConstraints* PerfMonitor::get_original_constraints() { auto* new_constraints = new PerfConstraints(false, config->sample_interval, - config->pkt_cnt); + config->pkt_cnt, config->flow_ip_all); return new_constraints; }