From: Maya Dagon (mdagon) Date: Tue, 13 May 2025 14:44:04 +0000 (+0000) Subject: Pull request #4728: extractor: support conn.log history field X-Git-Tag: 3.8.1.0~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1250a16cca179a0e4555f2bdfdf27d118906f8a3;p=thirdparty%2Fsnort3.git Pull request #4728: extractor: support conn.log history field Merge in SNORT/snort3 from ~MDAGON/snort3:conn_state to master Squashed commit of the following: commit dbce4ec8618a4d3e0ecda6fa4d4375de06eee9c0 Author: maya dagon Date: Wed Apr 30 13:56:59 2025 -0400 extractor: support conn.log history field --- diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt index 5d36e36ca..0ac8b4d6d 100644 --- a/doc/user/extractor.txt +++ b/doc/user/extractor.txt @@ -157,6 +157,29 @@ Fields supported for connection: For TCP orig_bytes and resp_bytes are calculated using first seen sequence number and next expected sequence number. These are reset during TCP flow restart. For this case only bytes seen following the restart will be reported. +* `conn_state` - records the connection state, which varies depending on the protocol (UDP, TCP, or others): + +UDP Connection States: + + ** CLT_SRV_UDP_SEEN: Packets were seen from both the client and server. + ** CLT_UDP_SEEN: Only client packets were observed. + ** SRV_UDP_SEEN: Only server packets were observed. + +TCP Connection States: + +The TCP connection state tracks both client and server states, each prefixed with CLT_ (for the client) and SRV_ (for the server). +These states follow the TCP state machine as defined by the RFC, with the addition of TCP_MID_STREAM_SENT and TCP_MID_STREAM_REC to handle mid-stream traffic and TCP_STATE_NONE. + +OTH (Other Traffic): + +The OTH state is used for all non-UDP and non-TCP traffic, as well as for error cases. + +* `history` - a string that tracks the connection's history. It uses letters to represent events, with uppercase letters denoting client-side events and lowercase letters for server-side events. Each letter appears only once for each direction, regardless of how many times the event occurs. + +UDP Events: d: Packet with payload. + +TCP Events: s: SYN, h: SYN-ACK, a: Pure ACK or PUSH, d: Packet with payload, f: FIN, r: Reset. + Fields supported for 'weird' and 'notice' logs: * `sid` - unique signature number of the rule diff --git a/doc/user/features.txt b/doc/user/features.txt index 823903bd4..f1e347894 100644 --- a/doc/user/features.txt +++ b/doc/user/features.txt @@ -97,7 +97,7 @@ include::pop_imap.txt[] include::port_scan.txt[] -=== Protocol Data Logging +=== Advanced Logging include::extractor.txt[] diff --git a/src/flow/flow.cc b/src/flow/flow.cc index 1ee03575b..06f17b2cb 100644 --- a/src/flow/flow.cc +++ b/src/flow/flow.cc @@ -36,6 +36,7 @@ #include "packet_io/packet_tracer.h" #include "protocols/packet.h" #include "protocols/tcp.h" +#include "pub_sub/eof_event.h" #include "pub_sub/intrinsic_event_ids.h" #include "sfip/sf_ip.h" #include "time/clock_defs.h" @@ -47,8 +48,9 @@ extern THREAD_LOCAL class FlowControl* flow_con; Flow::~Flow() { - DataBus::publish(intrinsic_pub_id, IntrinsicEventIds::FLOW_END, nullptr, this); - + EofEvent eof_event(this); + DataBus::publish(intrinsic_pub_id, IntrinsicEventIds::FLOW_END, eof_event, this); + free_flow_data(); delete session; diff --git a/src/network_inspectors/extractor/extractor_conn.cc b/src/network_inspectors/extractor/extractor_conn.cc index 129a0f122..35d10e80c 100644 --- a/src/network_inspectors/extractor/extractor_conn.cc +++ b/src/network_inspectors/extractor/extractor_conn.cc @@ -26,6 +26,7 @@ #include "detection/detection_engine.h" #include "flow/flow_key.h" #include "profiler/profiler.h" +#include "pub_sub/eof_event.h" #include "pub_sub/intrinsic_event_ids.h" #include "sfip/sf_ip.h" #include "stream/tcp/tcp_session.h" @@ -125,10 +126,22 @@ static const char* get_proto(const DataEvent*, const Flow* f) return (iter != pkttype_to_protocol.end()) ? iter->second.c_str() : ""; } +static const char* get_history(const DataEvent* event, const Flow*) +{ + return ((const EofEvent*)event)->get_history().c_str(); +} + +static const char* get_state(const DataEvent* event, const Flow*) +{ + return ((const EofEvent*)event)->get_state().c_str(); +} + static const map sub_buf_getters = { {"proto", get_proto}, - {"service", get_service} + {"service", get_service}, + {"history", get_history}, + {"conn_state", get_state} }; THREAD_LOCAL const snort::Connector::ID* ConnExtractor::log_id = nullptr; diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index aab7ae3e1..15fd67ab3 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -372,7 +372,9 @@ const ServiceBlueprint ConnExtractorService::blueprint = "resp_pkts", "duration", "orig_bytes", - "resp_bytes" + "resp_bytes", + "history", + "conn_state" }, }; diff --git a/src/pub_sub/CMakeLists.txt b/src/pub_sub/CMakeLists.txt index ca205d5ea..d85449e2d 100644 --- a/src/pub_sub/CMakeLists.txt +++ b/src/pub_sub/CMakeLists.txt @@ -10,6 +10,7 @@ set (PUB_SUB_INCLUDES detection_events.h dhcp_events.h domain_fronting.h + eof_event.h eve_process_event.h expect_events.h external_event_ids.h @@ -42,6 +43,7 @@ add_library( pub_sub OBJECT http_events.cc detection_events.cc dns_events.cc + eof_event.cc http_request_body_event.cc http_body_event.cc http_transaction_end_event.cc diff --git a/src/pub_sub/eof_event.cc b/src/pub_sub/eof_event.cc new file mode 100644 index 000000000..239a6daed --- /dev/null +++ b/src/pub_sub/eof_event.cc @@ -0,0 +1,175 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// eof_event.cc author Maya Dagon + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "eof_event.h" + +#include "stream/tcp/tcp_session.h" +#include "stream/udp/udp_session.h" + +using namespace snort; +using namespace std; + +//------------------------------------------------------------------------- +// History +//------------------------------------------------------------------------- + +static void one_side_history(const TcpSession* ssn, string& history, bool client) +{ + const auto& events = client ? ssn->tcp_ssn_stats.client_events : ssn->tcp_ssn_stats.server_events; + if (events.test(TcpStreamTracker::TcpEvent::TCP_SYN_SENT_EVENT)) + history += client ? "S" : "s"; + if (events.test(TcpStreamTracker::TcpEvent::TCP_SYN_ACK_SENT_EVENT)) + history += client ? "H" : "h"; + if (events.test(TcpStreamTracker::TcpEvent::TCP_ACK_SENT_EVENT)) + history += client ? "A" : "a"; + if (events.test(TcpStreamTracker::TcpEvent::TCP_DATA_SEG_SENT_EVENT)) + history += client ? "D" : "d"; + if (events.test(TcpStreamTracker::TcpEvent::TCP_FIN_SENT_EVENT)) + history += client ? "F" : "f"; + if (events.test(TcpStreamTracker::TcpEvent::TCP_RST_SENT_EVENT)) + history += client ? "R" : "r"; +} + +static void get_tcp_history(const TcpSession* ssn, string& history) +{ + one_side_history(ssn, history, true); + one_side_history(ssn, history, false); +} + +static void get_udp_history(const UdpSession* ssn, string& history) +{ + if (ssn->payload_bytes_seen_client) + history += "D"; + if (ssn->payload_bytes_seen_server) + history += "d"; +} + +const string& EofEvent::get_history() const +{ + history = ""; + + if (f->session == nullptr) + return history; + + if (f->pkt_type == PktType::TCP) + get_tcp_history((const TcpSession*)f->session, history); + else if (f->pkt_type == PktType::UDP) + get_udp_history((const UdpSession*)f->session, history); + + return history; +} + +//------------------------------------------------------------------------- +// State +//------------------------------------------------------------------------- + +static void get_udp_state(const Flow* f, string& state) +{ + static const string state_udp_clt = "CLT_UDP_SEEN"; + static const string state_udp_srv = "SRV_UDP_SEEN"; + static const string state_udp_both = "CLT_SRV_UDP_SEEN"; + + if (f->flowstats.client_pkts and f->flowstats.server_pkts) + state = state_udp_both; + else if (f->flowstats.client_pkts) + state = state_udp_clt; + else if (f->flowstats.server_pkts) + state = state_udp_srv; +} + +static void get_tcp_state(const Flow* f, string& state) +{ + const TcpSession* ssn = (TcpSession*) f->session; + if (ssn == nullptr) + return; + + static const string client_prefix = "CLT_"; + static const string server_prefix = "SRV_"; + + state = client_prefix + tcp_state_names[ssn->client.get_tcp_state()] + " " + + server_prefix + tcp_state_names[ssn->server.get_tcp_state()]; +} + +const string& EofEvent::get_state() const +{ + static const string state_oth = "OTH"; + state = state_oth; + + if (f->pkt_type == PktType::TCP) + get_tcp_state(f, state); + else if (f->pkt_type == PktType::UDP) + get_udp_state(f, state); + + return state; +} + +//------------------------------------------------------------------------- +// Unit Tests +//------------------------------------------------------------------------- + +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +TEST_CASE("coverage", "[eof_event]") +{ + Flow* flow = new Flow; + InspectionPolicy ins; + set_inspection_policy(&ins); + NetworkPolicy net; + set_network_policy(&net); + EofEvent eof(flow); + + SECTION("history no ssn") + { + const string& history = eof.get_history(); + CHECK(history == ""); + } + + SECTION("tcp state no ssn") + { + flow->pkt_type = PktType::TCP; + const string& state = eof.get_state(); + CHECK(state == "OTH"); + } + + SECTION("udp state OTH") + { + flow->pkt_type = PktType::UDP; + const string& state = eof.get_state(); + CHECK(state == "OTH"); + } + + SECTION("udp state SRV_UDP_SEEN") + { + flow->flowstats.server_pkts = 1; + flow->pkt_type = PktType::UDP; + const string& state = eof.get_state(); + CHECK(state == "SRV_UDP_SEEN"); + } + + delete flow; +} + +#endif + diff --git a/src/pub_sub/eof_event.h b/src/pub_sub/eof_event.h new file mode 100644 index 000000000..1b5dccbb5 --- /dev/null +++ b/src/pub_sub/eof_event.h @@ -0,0 +1,42 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025-2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// eof_event.h author Maya Dagon + +#ifndef EOF_EVENT_H +#define EOF_EVENT_H + +#include "framework/data_bus.h" + +namespace snort +{ +class SO_PUBLIC EofEvent : public snort::DataEvent +{ +public: + EofEvent(const Flow* const flow) : f(flow) { } + const std::string& get_history() const; + const std::string& get_state() const; + +private: + const Flow* const f; + mutable std::string history; + mutable std::string state; +}; +} + +#endif diff --git a/src/stream/tcp/tcp_session.h b/src/stream/tcp/tcp_session.h index 2e4de5705..88c0dc3d3 100644 --- a/src/stream/tcp/tcp_session.h +++ b/src/stream/tcp/tcp_session.h @@ -139,6 +139,14 @@ public: uint8_t held_packet_dir = SSN_DIR_NONE; uint8_t ecn = 0; + struct TcpSessionStats + { + using TcpEvents = std::bitset; + TcpEvents client_events; + TcpEvents server_events; + }; + TcpSessionStats tcp_ssn_stats; + private: int process_tcp_packet(TcpSegmentDescriptor&, const snort::Packet*); void set_os_policy(); diff --git a/src/stream/tcp/tcp_stream_tracker.cc b/src/stream/tcp/tcp_stream_tracker.cc index 054e47c54..c45425548 100644 --- a/src/stream/tcp/tcp_stream_tracker.cc +++ b/src/stream/tcp/tcp_stream_tracker.cc @@ -53,7 +53,7 @@ const std::list::iterator TcpStreamTracker::null_iterator { }; const char* tcp_state_names[] = { "TCP_LISTEN", "TCP_SYN_SENT", "TCP_SYN_RECV", - "TCP_ESTABLISHED", + "TCP_ESTABLISHED", "TCP_MID_STREAM_SENT", "TCP_MID_STREAM_RECV", "TCP_FIN_WAIT1", "TCP_FIN_WAIT2", "TCP_CLOSE_WAIT", "TCP_CLOSING", "TCP_LAST_ACK", "TCP_TIME_WAIT", "TCP_CLOSED", "TCP_STATE_NONE" @@ -171,6 +171,9 @@ TcpStreamTracker::TcpEvent TcpStreamTracker::set_tcp_event(const TcpSegmentDescr tcpStats.no_flags_set++; tcp_event = TCP_NO_FLAGS_EVENT; } + + (client_tracker) ? session->tcp_ssn_stats.client_events.set(tcp_event) : + session->tcp_ssn_stats.server_events.set(tcp_event); } else { diff --git a/src/stream/tcp/tcp_stream_tracker.h b/src/stream/tcp/tcp_stream_tracker.h index 83d2261dc..c30c01f18 100644 --- a/src/stream/tcp/tcp_stream_tracker.h +++ b/src/stream/tcp/tcp_stream_tracker.h @@ -72,18 +72,19 @@ public: enum TcpEvent : uint8_t { TCP_SYN_SENT_EVENT, - TCP_SYN_RECV_EVENT, TCP_SYN_ACK_SENT_EVENT, - TCP_SYN_ACK_RECV_EVENT, TCP_ACK_SENT_EVENT, - TCP_ACK_RECV_EVENT, TCP_DATA_SEG_SENT_EVENT, - TCP_DATA_SEG_RECV_EVENT, TCP_FIN_SENT_EVENT, - TCP_FIN_RECV_EVENT, TCP_RST_SENT_EVENT, - TCP_RST_RECV_EVENT, TCP_NO_FLAGS_EVENT, + TCP_MAX_TALKER_EVENT = TCP_NO_FLAGS_EVENT, + TCP_SYN_RECV_EVENT, + TCP_SYN_ACK_RECV_EVENT, + TCP_ACK_RECV_EVENT, + TCP_DATA_SEG_RECV_EVENT, + TCP_FIN_RECV_EVENT, + TCP_RST_RECV_EVENT, TCP_MAX_EVENTS };